84#include "llvm/IR/IntrinsicsARM.h"
122#define DEBUG_TYPE "arm-isel"
125STATISTIC(NumMovwMovt,
"Number of GAs materialized with movw + movt");
126STATISTIC(NumLoopByVals,
"Number of loops generated for byval arguments");
128 "Number of constants with their storage promoted into constant pools");
132 cl::desc(
"Enable / disable ARM interworking (for debugging only)"),
137 cl::desc(
"Enable / disable promotion of unnamed_addr constants into "
142 cl::desc(
"Maximum size of constant to promote into a constant pool"),
146 cl::desc(
"Maximum size of ALL constants to promote into a constant pool"),
151 cl::desc(
"Maximum interleave factor for MVE VLDn to generate."),
156 ARM::R0, ARM::R1, ARM::R2, ARM::R3
170void ARMTargetLowering::addTypeForNEON(
MVT VT,
MVT PromotedLdStVT) {
171 if (VT != PromotedLdStVT) {
180 if (ElemTy != MVT::f64)
184 if (ElemTy == MVT::i32) {
228void ARMTargetLowering::addDRTypeForNEON(
MVT VT) {
230 addTypeForNEON(VT, MVT::f64);
233void ARMTargetLowering::addQRTypeForNEON(
MVT VT) {
235 addTypeForNEON(VT, MVT::v2f64);
238void ARMTargetLowering::setAllExpand(
MVT VT) {
251void ARMTargetLowering::addAllExtLoads(
const MVT From,
const MVT To,
258void ARMTargetLowering::addMVEVectorTypes(
bool HasMVEFP) {
259 const MVT IntTypes[] = { MVT::v16i8, MVT::v8i16, MVT::v4i32 };
261 for (
auto VT : IntTypes) {
335 const MVT FloatTypes[] = { MVT::v8f16, MVT::v4f32 };
336 for (
auto VT : FloatTypes) {
405 const MVT LongTypes[] = { MVT::v2i64, MVT::v2f64 };
406 for (
auto VT : LongTypes) {
423 addAllExtLoads(MVT::v8i16, MVT::v8i8,
Legal);
424 addAllExtLoads(MVT::v4i32, MVT::v4i16,
Legal);
425 addAllExtLoads(MVT::v4i32, MVT::v4i8,
Legal);
442 for (
auto VT : {MVT::v8i8, MVT::v4i8, MVT::v4i16}) {
451 const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1};
452 for (
auto VT : pTypes) {
508 for (
int LCID = 0; LCID < RTLIB::UNKNOWN_LIBCALL; ++LCID)
516 if (Subtarget->isThumb() && Subtarget->
hasVFP2Base() &&
517 Subtarget->
hasARMOps() && !Subtarget->useSoftFloat()) {
518 static const struct {
520 const char *
const Name;
542 { RTLIB::UO_F32,
"__unordsf2vfp",
ISD::SETNE },
551 { RTLIB::UO_F64,
"__unorddf2vfp",
ISD::SETNE },
576 for (
const auto &LC : LibraryCalls) {
588 static const struct {
590 const char *
const Name;
591 const CallingConv::ID
CC;
675 for (
const auto &LC : LibraryCalls) {
685 static const struct {
687 const char *
const Name;
688 const CallingConv::ID
CC;
690 } MemOpsLibraryCalls[] = {
698 for (
const auto &LC : MemOpsLibraryCalls) {
708 static const struct {
710 const char *
const Name;
711 const CallingConv::ID
CC;
723 for (
const auto &LC : LibraryCalls) {
755 static const struct {
757 const char *
const Name;
758 const CallingConv::ID
CC;
765 for (
const auto &LC : LibraryCalls) {
776 if (!Subtarget->useSoftFloat() && !Subtarget->
isThumb1Only() &&
777 Subtarget->hasFPRegs()) {
787 setAllExpand(MVT::f32);
788 if (!Subtarget->hasFP64())
789 setAllExpand(MVT::f64);
792 if (Subtarget->hasFullFP16()) {
801 if (Subtarget->hasBF16()) {
803 setAllExpand(MVT::bf16);
804 if (!Subtarget->hasFullFP16())
811 addAllExtLoads(VT, InnerVT,
Expand);
826 if (Subtarget->hasMVEIntegerOps())
827 addMVEVectorTypes(Subtarget->hasMVEFloatOps());
830 if (Subtarget->hasLOB()) {
834 if (Subtarget->hasNEON()) {
835 addDRTypeForNEON(MVT::v2f32);
836 addDRTypeForNEON(MVT::v8i8);
837 addDRTypeForNEON(MVT::v4i16);
838 addDRTypeForNEON(MVT::v2i32);
839 addDRTypeForNEON(MVT::v1i64);
841 addQRTypeForNEON(MVT::v4f32);
842 addQRTypeForNEON(MVT::v2f64);
843 addQRTypeForNEON(MVT::v16i8);
844 addQRTypeForNEON(MVT::v8i16);
845 addQRTypeForNEON(MVT::v4i32);
846 addQRTypeForNEON(MVT::v2i64);
848 if (Subtarget->hasFullFP16()) {
849 addQRTypeForNEON(MVT::v8f16);
850 addDRTypeForNEON(MVT::v4f16);
853 if (Subtarget->hasBF16()) {
854 addQRTypeForNEON(MVT::v8bf16);
855 addDRTypeForNEON(MVT::v4bf16);
859 if (Subtarget->hasMVEIntegerOps() || Subtarget->hasNEON()) {
899 if (Subtarget->hasNEON()) {
1013 for (
MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
1022 for (
auto VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16,
1031 if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) {
1039 if (Subtarget->hasMVEIntegerOps()) {
1044 if (Subtarget->hasMVEFloatOps()) {
1048 if (!Subtarget->hasFP64()) {
1095 if (Subtarget->hasFullFP16()) {
1101 if (!Subtarget->hasFP16()) {
1149 if (Subtarget->hasDSP()) {
1171 if (Subtarget->
isThumb1Only() || !Subtarget->hasV6Ops()
1172 || (Subtarget->
isThumb2() && !Subtarget->hasDSP()))
1187 if (Subtarget->hasMVEIntegerOps())
1197 if (!Subtarget->
isThumb1Only() && Subtarget->hasV6T2Ops())
1208 if (!Subtarget->hasV5TOps() || Subtarget->
isThumb1Only()) {
1217 if (Subtarget->hasPerfMon())
1221 if (!Subtarget->hasV6Ops())
1224 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
1225 : Subtarget->hasDivideInARMMode();
1232 if (Subtarget->
isTargetWindows() && !Subtarget->hasDivideInThumbMode()) {
1249 HasStandaloneRem =
false;
1254 const char *
const Name;
1255 const CallingConv::ID
CC;
1256 } LibraryCalls[] = {
1268 for (
const auto &LC : LibraryCalls) {
1275 const char *
const Name;
1276 const CallingConv::ID
CC;
1277 } LibraryCalls[] = {
1289 for (
const auto &LC : LibraryCalls) {
1327 InsertFencesForAtomic =
false;
1329 (!Subtarget->isThumb() || Subtarget->hasV8MBaselineOps())) {
1333 if (!Subtarget->isThumb() || !Subtarget->
isMClass())
1338 if (!Subtarget->hasAcquireRelease() ||
1341 InsertFencesForAtomic =
true;
1347 if (Subtarget->hasDataBarrier())
1348 InsertFencesForAtomic =
true;
1368 if (!InsertFencesForAtomic) {
1376 (!Subtarget->
isMClass() && Subtarget->hasV6Ops())) {
1388 }
else if ((Subtarget->
isMClass() && Subtarget->hasV8MBaselineOps()) ||
1389 Subtarget->hasForced32BitAtomics()) {
1403 if (!Subtarget->hasV6Ops()) {
1409 if (!Subtarget->useSoftFloat() && Subtarget->hasFPRegs() &&
1441 if (Subtarget->hasFullFP16()) {
1451 if (Subtarget->hasFullFP16())
1466 if (!Subtarget->useSoftFloat() && Subtarget->
hasVFP2Base() &&
1480 if (!Subtarget->useSoftFloat() && !Subtarget->
isThumb1Only()) {
1488 if (!Subtarget->hasFP16()) {
1519 if (Subtarget->hasNEON()) {
1526 if (Subtarget->hasFP64()) {
1539 if (Subtarget->hasFullFP16()) {
1558 if (Subtarget->hasNEON()) {
1570 if (Subtarget->hasFullFP16()) {
1602 if (Subtarget->hasMVEIntegerOps())
1605 if (Subtarget->hasV6Ops())
1610 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) ||
1617 if (Subtarget->useSoftFloat() || Subtarget->
isThumb1Only() ||
1646 return Subtarget->useSoftFloat();
1659std::pair<const TargetRegisterClass *, uint8_t>
1670 case MVT::f32:
case MVT::f64:
case MVT::v8i8:
case MVT::v4i16:
1671 case MVT::v2i32:
case MVT::v1i64:
case MVT::v2f32:
1672 RRC = &ARM::DPRRegClass;
1680 case MVT::v16i8:
case MVT::v8i16:
case MVT::v4i32:
case MVT::v2i64:
1681 case MVT::v4f32:
case MVT::v2f64:
1682 RRC = &ARM::DPRRegClass;
1686 RRC = &ARM::DPRRegClass;
1690 RRC = &ARM::DPRRegClass;
1694 return std::make_pair(RRC,
Cost);
1698#define MAKE_CASE(V) \
1917 if ((Subtarget->hasMVEIntegerOps() &&
1918 (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 ||
1919 VT == MVT::v16i8)) ||
1920 (Subtarget->hasMVEFloatOps() &&
1921 (VT == MVT::v2f64 || VT == MVT::v4f32 || VT == MVT::v8f16)))
1935 if (Subtarget->hasNEON()) {
1936 if (VT == MVT::v4i64)
1937 return &ARM::QQPRRegClass;
1938 if (VT == MVT::v8i64)
1939 return &ARM::QQQQPRRegClass;
1941 if (Subtarget->hasMVEIntegerOps()) {
1942 if (VT == MVT::v4i64)
1943 return &ARM::MQQPRRegClass;
1944 if (VT == MVT::v8i64)
1945 return &ARM::MQQQQPRRegClass;
1954 Align &PrefAlign)
const {
1973 unsigned NumVals =
N->getNumValues();
1977 for (
unsigned i = 0; i != NumVals; ++i) {
1978 EVT VT =
N->getValueType(i);
1979 if (VT == MVT::Glue || VT == MVT::Other)
1985 if (!
N->isMachineOpcode())
2010 return Const->getZExtValue() == 16;
2018 return Const->getZExtValue() == 16;
2026 return Const->getZExtValue() == 16;
2094ARMTargetLowering::getEffectiveCallingConv(CallingConv::ID
CC,
2095 bool isVarArg)
const {
2116 else if (Subtarget->hasFPRegs() && !Subtarget->
isThumb1Only() &&
2137 bool isVarArg)
const {
2138 return CCAssignFnForNode(
CC,
false, isVarArg);
2142 bool isVarArg)
const {
2143 return CCAssignFnForNode(
CC,
true, isVarArg);
2148CCAssignFn *ARMTargetLowering::CCAssignFnForNode(CallingConv::ID
CC,
2150 bool isVarArg)
const {
2151 switch (getEffectiveCallingConv(
CC, isVarArg)) {
2177 if (Subtarget->hasFullFP16()) {
2182 Val = DAG.
getNode(ISD::BITCAST, dl, ValVT, Val);
2190 if (Subtarget->hasFullFP16()) {
2194 Val = DAG.
getNode(ISD::BITCAST, dl,
2199 return DAG.
getNode(ISD::BITCAST, dl, LocVT, Val);
2204SDValue ARMTargetLowering::LowerCallResult(
2205 SDValue Chain,
SDValue InGlue, CallingConv::ID CallConv,
bool isVarArg,
2208 SDValue ThisVal,
bool isCmseNSCall)
const {
2216 for (
unsigned i = 0; i != RVLocs.
size(); ++i) {
2221 if (i == 0 && isThisReturn) {
2223 "unexpected return calling convention register assignment");
2301std::pair<SDValue, MachinePointerInfo> ARMTargetLowering::computeAddrForCallArg(
2303 bool IsTailCall,
int SPDiff)
const {
2325 return std::make_pair(DstAddr, DstInfo);
2330 RegsToPassVector &RegsToPass,
2337 DAG.
getVTList(MVT::i32, MVT::i32), Arg);
2338 unsigned id = Subtarget->
isLittle() ? 0 : 1;
2351 std::tie(DstAddr, DstInfo) =
2352 computeAddrForCallArg(dl, DAG, NextVA, StackPtr, IsTailCall, SPDiff);
2377 CallingConv::ID CallConv = CLI.
CallConv;
2384 bool isStructRet = (Outs.
empty()) ?
false : Outs[0].Flags.isSRet();
2385 bool isThisReturn =
false;
2386 bool isCmseNSCall =
false;
2387 bool isSibCall =
false;
2388 bool PreferIndirect =
false;
2389 bool GuardWithBTI =
false;
2399 !Subtarget->noBTIAtReturnTwice())
2404 isCmseNSCall =
true;
2423 PreferIndirect = Subtarget->isThumb() && Subtarget->
hasMinSize() &&
2425 return isa<Instruction>(U) &&
2426 cast<Instruction>(U)->getParent() == BB;
2433 IsEligibleForTailCallOptimization(CLI, CCInfo, ArgLocs, PreferIndirect);
2447 "site marked musttail");
2450 unsigned NumBytes = CCInfo.getStackSize();
2459 if (isTailCall && !isSibCall) {
2466 assert(StackAlign &&
"data layout string is missing stack alignment");
2467 NumBytes =
alignTo(NumBytes, *StackAlign);
2472 SPDiff = NumReusableBytes - NumBytes;
2476 if (SPDiff < 0 && AFI->getArgRegsSaveSize() < (
unsigned)-SPDiff)
2492 RegsToPassVector RegsToPass;
2500 bool AfterFormalArgLoads =
false;
2504 for (
unsigned i = 0, realArgIdx = 0, e = ArgLocs.
size();
2506 ++i, ++realArgIdx) {
2508 SDValue Arg = OutVals[realArgIdx];
2510 bool isByVal = Flags.isByVal();
2530 if (isTailCall && VA.
isMemLoc() && !AfterFormalArgLoads) {
2532 AfterFormalArgLoads =
true;
2544 auto ArgVT = Outs[realArgIdx].ArgVT;
2545 if (isCmseNSCall && (ArgVT == MVT::f16)) {
2563 PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass, VA, ArgLocs[++i],
2564 StackPtr, MemOpChains, isTailCall, SPDiff);
2568 PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass, VA, ArgLocs[++i],
2569 StackPtr, MemOpChains, isTailCall, SPDiff);
2574 std::tie(DstAddr, DstInfo) =
2575 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2579 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i],
2580 StackPtr, MemOpChains, isTailCall, SPDiff);
2582 if (realArgIdx == 0 && Flags.isReturned() && !Flags.isSwiftSelf() &&
2583 Outs[0].VT == MVT::i32) {
2585 "unexpected calling convention register assignment");
2586 assert(!
Ins.empty() && Ins[0].VT == MVT::i32 &&
2587 "unexpected use of 'returned'");
2588 isThisReturn =
true;
2592 CSInfo.ArgRegPairs.emplace_back(VA.
getLocReg(), i);
2593 RegsToPass.push_back(std::make_pair(VA.
getLocReg(), Arg));
2594 }
else if (isByVal) {
2596 unsigned offset = 0;
2600 unsigned ByValArgsCount = CCInfo.getInRegsParamsCount();
2601 unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
2603 if (CurByValIdx < ByValArgsCount) {
2605 unsigned RegBegin, RegEnd;
2606 CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd);
2611 for (i = 0, j = RegBegin;
j < RegEnd; i++,
j++) {
2618 RegsToPass.push_back(std::make_pair(j, Load));
2623 offset = RegEnd - RegBegin;
2625 CCInfo.nextInRegsParam();
2628 if (Flags.getByValSize() > 4*offset) {
2632 std::tie(Dst, DstInfo) =
2633 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2639 DAG.
getConstant(Flags.getNonZeroByValAlign().value(), dl, MVT::i32);
2642 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
2650 std::tie(DstAddr, DstInfo) =
2651 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2658 if (!MemOpChains.
empty())
2664 for (
unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
2665 Chain = DAG.
getCopyToReg(Chain, dl, RegsToPass[i].first,
2666 RegsToPass[i].second, InGlue);
2673 bool isDirect =
false;
2678 GVal =
G->getGlobal();
2679 bool isStub = !
TM.shouldAssumeDSOLocal(GVal) && Subtarget->
isTargetMachO();
2681 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->
isMClass());
2682 bool isLocalARMFunc =
false;
2685 if (Subtarget->genLongCalls()) {
2687 "long-calls codegen is not position independent!");
2692 if (Subtarget->genExecuteOnly()) {
2711 const char *
Sym = S->getSymbol();
2713 if (Subtarget->genExecuteOnly()) {
2733 if (!PreferIndirect) {
2738 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !
ARMInterworking);
2740 if (isStub && Subtarget->
isThumb1Only() && !Subtarget->hasV5TOps()) {
2752 "Windows is the only supported COFF target");
2756 else if (!
TM.shouldAssumeDSOLocal(GVal))
2772 const char *
Sym = S->getSymbol();
2773 if (isARMFunc && Subtarget->
isThumb1Only() && !Subtarget->hasV5TOps()) {
2777 ARMPCLabelIndex, 4);
2791 assert(!isARMFunc && !isDirect &&
2792 "Cannot handle call to ARM function or direct call");
2795 "call to non-secure function would "
2796 "require passing arguments on stack",
2803 "call to non-secure function would return value through pointer",
2811 if (Subtarget->isThumb()) {
2814 else if (isCmseNSCall)
2816 else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
2821 if (!isDirect && !Subtarget->hasV5TOps())
2823 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() &&
2836 if (isTailCall && !isSibCall) {
2841 std::vector<SDValue> Ops;
2842 Ops.push_back(Chain);
2843 Ops.push_back(Callee);
2852 for (
unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
2853 Ops.push_back(DAG.
getRegister(RegsToPass[i].first,
2854 RegsToPass[i].second.getValueType()));
2866 isThisReturn =
false;
2872 assert(Mask &&
"Missing call preserved mask for calling convention");
2876 Ops.push_back(InGlue);
2888 Chain = DAG.
getNode(CallOpc, dl, NodeTys, Ops);
2900 Chain = DAG.
getCALLSEQ_END(Chain, NumBytes, CalleePopBytes, InGlue, dl);
2906 return LowerCallResult(Chain, InGlue, CallConv, isVarArg, Ins, dl, DAG,
2907 InVals, isThisReturn,
2908 isThisReturn ? OutVals[0] :
SDValue(), isCmseNSCall);
2915void ARMTargetLowering::HandleByVal(
CCState *State,
unsigned &
Size,
2916 Align Alignment)
const {
2918 Alignment = std::max(Alignment,
Align(4));
2924 unsigned AlignInRegs = Alignment.
value() / 4;
2925 unsigned Waste = (ARM::R4 -
Reg) % AlignInRegs;
2926 for (
unsigned i = 0; i < Waste; ++i)
2932 unsigned Excess = 4 * (ARM::R4 -
Reg);
2938 const unsigned NSAAOffset = State->getStackSize();
2939 if (NSAAOffset != 0 &&
Size > Excess) {
2951 unsigned ByValRegBegin =
Reg;
2952 unsigned ByValRegEnd = std::min<unsigned>(Reg +
Size / 4, ARM::R4);
2953 State->addInRegsParamInfo(ByValRegBegin, ByValRegEnd);
2956 for (
unsigned i = Reg + 1; i != ByValRegEnd; ++i)
2962 Size = std::max<int>(
Size - Excess, 0);
2973 int FI = std::numeric_limits<int>::max();
2981 if (!Flags.isByVal()) {
2988 if (Flags.isByVal())
3003 assert(FI != std::numeric_limits<int>::max());
3014bool ARMTargetLowering::IsEligibleForTailCallOptimization(
3017 CallingConv::ID CalleeCC = CLI.
CallConv;
3042 for (
Register R : {ARM::R0, ARM::R1, ARM::R2, ARM::R3})
3043 AddressRegisters.
insert(R);
3046 AddressRegisters.
insert(ARM::R12);
3049 AddressRegisters.
erase(
AL.getLocReg());
3050 if (AddressRegisters.
empty())
3064 return CalleeCC == CallerCC;
3068 bool isCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
3070 if (isCalleeStructRet || isCallerStructRet)
3084 (!
TT.isOSWindows() ||
TT.isOSBinFormatELF() ||
TT.isOSBinFormatMachO()))
3091 getEffectiveCallingConv(CalleeCC, isVarArg),
3092 getEffectiveCallingConv(CallerCC, CallerF.
isVarArg()), MF,
C, Ins,
3098 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
3099 if (CalleeCC != CallerCC) {
3100 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
3101 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
3114 if (!Outs.
empty()) {
3121 for (
unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
3123 ++i, ++realArgIdx) {
3126 SDValue Arg = OutVals[realArgIdx];
3130 if (VA.
needsCustom() && (RegVT == MVT::f64 || RegVT == MVT::v2f64)) {
3137 if (!ArgLocs[++i].isRegLoc())
3139 if (RegVT == MVT::v2f64) {
3140 if (!ArgLocs[++i].isRegLoc())
3142 if (!ArgLocs[++i].isRegLoc())
3162ARMTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
3167 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
3176 StringRef IntKind =
F.getFnAttribute(
"interrupt").getValueAsString();
3189 if (IntKind ==
"" || IntKind ==
"IRQ" || IntKind ==
"FIQ" ||
3192 else if (IntKind ==
"SWI" || IntKind ==
"UNDEF")
3196 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF");
3205ARMTargetLowering::LowerReturn(
SDValue Chain, CallingConv::ID CallConv,
3223 bool isLittleEndian = Subtarget->
isLittle();
3235 "secure entry function would return value through pointer",
3241 for (
unsigned i = 0, realRVLocIdx = 0;
3243 ++i, ++realRVLocIdx) {
3247 SDValue Arg = OutVals[realRVLocIdx];
3248 bool ReturnF16 =
false;
3283 auto RetVT = Outs[realRVLocIdx].ArgVT;
3305 DAG.
getVTList(MVT::i32, MVT::i32), Half);
3309 HalfGPRs.
getValue(isLittleEndian ? 0 : 1), Glue);
3315 HalfGPRs.
getValue(isLittleEndian ? 1 : 0), Glue);
3327 DAG.
getVTList(MVT::i32, MVT::i32), Arg);
3329 fmrrd.
getValue(isLittleEndian ? 0 : 1), Glue);
3334 fmrrd.
getValue(isLittleEndian ? 1 : 0), Glue);
3378 return DAG.
getNode(RetNode, dl, MVT::Other, RetOps);
3381bool ARMTargetLowering::isUsedByReturnOnly(
SDNode *
N,
SDValue &Chain)
const {
3382 if (
N->getNumValues() != 1)
3384 if (!
N->hasNUsesOfValue(1, 0))
3392 if (
Copy->getOperand(
Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3394 TCChain =
Copy->getOperand(0);
3416 if (U->getOperand(U->getNumOperands() - 1).getValueType() == MVT::Glue)
3422 }
else if (
Copy->getOpcode() == ISD::BITCAST) {
3424 if (!
Copy->hasOneUse())
3431 if (
Copy->getOperand(
Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3433 TCChain =
Copy->getOperand(0);
3438 bool HasRet =
false;
3453bool ARMTargetLowering::mayBeEmittedAsTailCall(
const CallInst *CI)
const {
3471 &&
"LowerWRITE_REGISTER called for non-i64 type argument.");
3487 EVT PtrVT =
Op.getValueType();
3497 if (Subtarget->genExecuteOnly()) {
3499 auto T =
const_cast<Type*
>(
CP->getType());
3500 auto C =
const_cast<Constant*
>(
CP->getConstVal());
3511 return LowerGlobalAddress(GA, DAG);
3516 Align CPAlign =
CP->getAlign();
3518 CPAlign = std::max(CPAlign,
Align(4));
3519 if (
CP->isMachineConstantPoolEntry())
3531 if (Subtarget->genExecuteOnly() && !Subtarget->hasV8MBaselineOps())
3540 unsigned ARMPCLabelIndex = 0;
3546 if (!IsPositionIndependent) {
3549 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
3560 if (!IsPositionIndependent)
3591ARMTargetLowering::LowerGlobalTLSAddressDarwin(
SDValue Op,
3594 "This function expects a Darwin target");
3599 SDValue DescAddr = LowerGlobalAddressDarwin(
Op, DAG);
3605 MVT::i32,
DL, Chain, DescAddr,
3629 Chain, FuncTLVGet, DAG.
getRegister(ARM::R0, MVT::i32),
3635ARMTargetLowering::LowerGlobalTLSAddressWindows(
SDValue Op,
3652 DAG.
getVTList(MVT::i32, MVT::Other), Ops);
3696 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3718 Args.push_back(Entry);
3726 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
3727 return CallResult.first;
3749 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3757 PtrVT, dl, Chain,
Offset,
3765 PtrVT, dl, Chain,
Offset,
3775 PtrVT, dl, Chain,
Offset,
3791 return LowerGlobalTLSAddressDarwin(
Op, DAG);
3794 return LowerGlobalTLSAddressWindows(
Op, DAG);
3803 return LowerToTLSGeneralDynamicModel(GA, DAG);
3806 return LowerToTLSExecModels(GA, DAG, model);
3815 while (!Worklist.
empty()) {
3823 if (!
I ||
I->getParent()->getParent() !=
F)
3852 if (!GVar || !GVar->hasInitializer() ||
3853 !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() ||
3854 !GVar->hasLocalLinkage())
3859 auto *
Init = GVar->getInitializer();
3861 Init->needsDynamicRelocation())
3873 unsigned RequiredPadding = 4 - (
Size % 4);
3874 bool PaddingPossible =
3875 RequiredPadding == 4 || (CDAInit && CDAInit->isString());
3880 unsigned PaddedSize =
Size + ((RequiredPadding == 4) ? 0 : RequiredPadding);
3904 if (RequiredPadding != 4) {
3909 while (RequiredPadding--)
3921 ++NumConstpoolPromoted;
3927 if (!(GV = GA->getAliaseeObject()))
3930 return V->isConstant();
3939 return LowerGlobalAddressWindows(
Op, DAG);
3941 return LowerGlobalAddressELF(
Op, DAG);
3943 return LowerGlobalAddressDarwin(
Op, DAG);
3955 if (GV->
isDSOLocal() && !Subtarget->genExecuteOnly())
3968 }
else if (Subtarget->
isROPI() && IsRO) {
3973 }
else if (Subtarget->
isRWPI() && !IsRO) {
3998 if (Subtarget->
useMovt() || Subtarget->genExecuteOnly()) {
4017 "ROPI/RWPI not currently supported for Darwin");
4043 "Windows on ARM expects to use movw/movt");
4045 "ROPI/RWPI not currently supported for Windows");
4052 else if (!
TM.shouldAssumeDSOLocal(GV))
4076 DAG.
getVTList(MVT::i32, MVT::Other),
Op.getOperand(0),
4077 Op.getOperand(1), Val);
4094SDValue ARMTargetLowering::LowerINTRINSIC_VOID(
4097 Op.getConstantOperandVal(
Op.getOperand(0).getValueType() == MVT::Other);
4101 case Intrinsic::arm_gnu_eabi_mcount: {
4110 assert(Mask &&
"Missing call preserved mask for calling convention");
4115 constexpr EVT ResultTys[] = {MVT::Other, MVT::Glue};
4119 if (Subtarget->isThumb())
4122 ARM::tBL_PUSHLR, dl, ResultTys,
4123 {ReturnAddress, DAG.getTargetConstant(ARMCC::AL, dl, PtrVT),
4124 DAG.getRegister(0, PtrVT), Callee, RegisterMask, Chain}),
4128 {ReturnAddress, Callee, RegisterMask, Chain}),
4137 unsigned IntNo =
Op.getConstantOperandVal(0);
4141 case Intrinsic::thread_pointer: {
4145 case Intrinsic::arm_cls: {
4146 const SDValue &Operand =
Op.getOperand(1);
4147 const EVT VTy =
Op.getValueType();
4158 case Intrinsic::arm_cls64: {
4161 const SDValue &Operand =
Op.getOperand(1);
4162 const EVT VTy =
Op.getValueType();
4185 case Intrinsic::eh_sjlj_lsda: {
4192 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0;
4202 if (IsPositionIndependent) {
4208 case Intrinsic::arm_neon_vabs:
4211 case Intrinsic::arm_neon_vabds:
4212 if (
Op.getValueType().isInteger())
4214 Op.getOperand(1),
Op.getOperand(2));
4216 case Intrinsic::arm_neon_vabdu:
4218 Op.getOperand(1),
Op.getOperand(2));
4219 case Intrinsic::arm_neon_vmulls:
4220 case Intrinsic::arm_neon_vmullu: {
4221 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls)
4224 Op.getOperand(1),
Op.getOperand(2));
4226 case Intrinsic::arm_neon_vminnm:
4227 case Intrinsic::arm_neon_vmaxnm: {
4228 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm)
4229 ? ISD::FMINNUM : ISD::FMAXNUM;
4231 Op.getOperand(1),
Op.getOperand(2));
4233 case Intrinsic::arm_neon_vminu:
4234 case Intrinsic::arm_neon_vmaxu: {
4235 if (
Op.getValueType().isFloatingPoint())
4237 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu)
4240 Op.getOperand(1),
Op.getOperand(2));
4242 case Intrinsic::arm_neon_vmins:
4243 case Intrinsic::arm_neon_vmaxs: {
4245 if (!
Op.getValueType().isFloatingPoint()) {
4246 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4249 Op.getOperand(1),
Op.getOperand(2));
4251 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4252 ? ISD::FMINIMUM : ISD::FMAXIMUM;
4254 Op.getOperand(1),
Op.getOperand(2));
4256 case Intrinsic::arm_neon_vtbl1:
4258 Op.getOperand(1),
Op.getOperand(2));
4259 case Intrinsic::arm_neon_vtbl2:
4261 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4262 case Intrinsic::arm_mve_pred_i2v:
4263 case Intrinsic::arm_mve_pred_v2i:
4266 case Intrinsic::arm_mve_vreinterpretq:
4269 case Intrinsic::arm_mve_lsll:
4271 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4272 case Intrinsic::arm_mve_asrl:
4274 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4285 if (!Subtarget->hasDataBarrier()) {
4289 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&
4290 "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!");
4301 }
else if (Subtarget->preferISHSTBarriers() &&
4310 DAG.
getConstant(Intrinsic::arm_dmb, dl, MVT::i32),
4318 (!Subtarget->
isThumb1Only() && Subtarget->hasV5TEOps())))
4320 return Op.getOperand(0);
4323 unsigned isRead =
~Op.getConstantOperandVal(2) & 1;
4325 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension()))
4327 return Op.getOperand(0);
4329 unsigned isData =
Op.getConstantOperandVal(4);
4330 if (Subtarget->isThumb()) {
4332 isRead = ~isRead & 1;
4333 isData = ~isData & 1;
4351 return DAG.
getStore(
Op.getOperand(0), dl, FR,
Op.getOperand(1),
4359 const SDLoc &dl)
const {
4365 RC = &ARM::tGPRRegClass;
4367 RC = &ARM::GPRRegClass;
4381 MVT::i32, dl, Root, FIN,
4402 const Value *OrigArg,
4403 unsigned InRegsParamRecordIdx,
4404 int ArgOffset,
unsigned ArgSize)
const {
4419 unsigned RBegin, REnd;
4429 ArgOffset = -4 * (ARM::R4 - RBegin);
4439 for (
unsigned Reg = RBegin, i = 0; Reg < REnd; ++
Reg, ++i) {
4448 if (!MemOps.
empty())
4457 unsigned TotalArgRegsSaveSize,
4458 bool ForceMutable)
const {
4469 CCInfo.
getStackSize(), std::max(4U, TotalArgRegsSaveSize));
4473bool ARMTargetLowering::splitValueIntoRegisterParts(
4475 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
4477 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4482 Val = DAG.
getNode(ISD::BITCAST,
DL, PartVT, Val);
4489SDValue ARMTargetLowering::joinRegisterPartsIntoValue(
4491 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
4492 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4499 Val = DAG.
getNode(ISD::BITCAST,
DL, ValueVT, Val);
4505SDValue ARMTargetLowering::LowerFormalArguments(
4506 SDValue Chain, CallingConv::ID CallConv,
bool isVarArg,
4521 unsigned CurArgIdx = 0;
4533 unsigned ArgRegBegin = ARM::R4;
4540 if (!Flags.isByVal())
4544 unsigned RBegin, REnd;
4546 ArgRegBegin = std::min(ArgRegBegin, RBegin);
4552 int lastInsIndex = -1;
4556 ArgRegBegin = std::min(ArgRegBegin, (
unsigned)
GPRArgRegs[RegIdx]);
4559 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin);
4563 for (
unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4565 if (Ins[VA.
getValNo()].isOrigArg()) {
4566 std::advance(CurOrigArg,
4567 Ins[VA.
getValNo()].getOrigArgIndex() - CurArgIdx);
4579 GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4586 MVT::f64, dl, Chain, FIN,
4589 ArgValue2 = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4597 ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4601 if (RegVT == MVT::f16 || RegVT == MVT::bf16)
4602 RC = &ARM::HPRRegClass;
4603 else if (RegVT == MVT::f32)
4604 RC = &ARM::SPRRegClass;
4605 else if (RegVT == MVT::f64 || RegVT == MVT::v4f16 ||
4606 RegVT == MVT::v4bf16)
4607 RC = &ARM::DPRRegClass;
4608 else if (RegVT == MVT::v2f64 || RegVT == MVT::v8f16 ||
4609 RegVT == MVT::v8bf16)
4610 RC = &ARM::QPRRegClass;
4611 else if (RegVT == MVT::i32)
4613 : &ARM::GPRRegClass;
4659 assert(VA.
getValVT() != MVT::i64 &&
"i64 should already be lowered");
4665 if (index != lastInsIndex)
4673 if (Flags.isByVal()) {
4674 assert(Ins[index].isOrigArg() &&
4675 "Byval arguments cannot be implicit");
4679 CCInfo, DAG, dl, Chain, &*CurOrigArg, CurByValIndex,
4694 lastInsIndex = index;
4701 VarArgStyleRegisters(CCInfo, DAG, dl, Chain, CCInfo.
getStackSize(),
4702 TotalArgRegsSaveSize);
4706 "secure entry function must not be variadic", dl.
getDebugLoc());
4717 assert(StackAlign &&
"data layout string is missing stack alignment");
4718 StackArgSize =
alignTo(StackArgSize, *StackAlign);
4727 "secure entry function requires arguments on stack", dl.
getDebugLoc());
4737 return CFP->getValueAPF().isPosZero();
4741 SDValue WrapperOp =
Op.getOperand(1).getOperand(0);
4744 return CFP->getValueAPF().isPosZero();
4746 }
else if (
Op->getOpcode() == ISD::BITCAST &&
4747 Op->getValueType(0) == MVT::f64) {
4762 const SDLoc &dl)
const {
4764 unsigned C = RHSC->getZExtValue();
4832 unsigned Mask = LHS.getConstantOperandVal(1);
4834 uint64_t RHSV = RHSC->getZExtValue();
4835 if (
isMask_32(Mask) && (RHSV & ~Mask) == 0 && Mask != 255 && Mask != 65535) {
4837 if (RHSV && (RHSV > 255 || (RHSV << ShiftBits) <= 255)) {
4839 LHS = DAG.
getNode(
ISD::SHL, dl, MVT::i32, LHS.getOperand(0), ShiftAmt);
4840 RHS = DAG.
getConstant(RHSV << ShiftBits, dl, MVT::i32);
4854 LHS.getConstantOperandVal(1) < 31) {
4895 return DAG.
getNode(CompareType, dl, MVT::Glue, LHS, RHS);
4901 bool Signaling)
const {
4902 assert(Subtarget->hasFP64() || RHS.getValueType() != MVT::f64);
4906 dl, MVT::Glue, LHS, RHS);
4909 dl, MVT::Glue, LHS);
4917 unsigned Opc =
Cmp.getOpcode();
4920 return DAG.
getNode(Opc,
DL, MVT::Glue,
Cmp.getOperand(0),
Cmp.getOperand(1));
4924 Opc =
Cmp.getOpcode();
4938std::pair<SDValue, SDValue>
4941 assert(
Op.getValueType() == MVT::i32 &&
"Unsupported value type");
4953 switch (
Op.getOpcode()) {
4966 DAG.
getVTList(
Op.getValueType(), MVT::i32), LHS, RHS)
5005 return std::make_pair(
Value, OverflowCmp);
5016 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Op, DAG, ARMcc);
5022 EVT VT =
Op.getValueType();
5025 ARMcc, CCR, OverflowCmp);
5065 EVT VT =
Op.getValueType();
5069 switch (
Op.getOpcode()) {
5094 EVT VT =
Op.getValueType();
5095 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP() || Subtarget->
isThumb1Only())
5105 switch (
Op->getOpcode()) {
5121 switch (
Op->getOpcode()) {
5140 DAG.
getNode(NewOpcode, dl, MVT::i32,
5151 unsigned Opc =
Cond.getOpcode();
5153 if (
Cond.getResNo() == 1 &&
5161 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Cond, DAG, ARMcc);
5163 EVT VT =
Op.getValueType();
5165 return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, CCR,
5180 if (CMOVTrue && CMOVFalse) {
5186 if (CMOVTrueVal == 1 && CMOVFalseVal == 0) {
5188 False = SelectFalse;
5189 }
else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) {
5195 EVT VT =
Op.getValueType();
5200 return getCMOV(dl, VT, True, False, ARMcc, CCR, Cmp, DAG);
5216 bool &swpCmpOps,
bool &swpVselOps) {
5244 swpCmpOps = !swpCmpOps;
5245 swpVselOps = !swpVselOps;
5268 if (!Subtarget->hasFP64() && VT == MVT::f64) {
5270 DAG.
getVTList(MVT::i32, MVT::i32), FalseVal);
5272 DAG.
getVTList(MVT::i32, MVT::i32), TrueVal);
5282 ARMcc, CCR, duplicateCmp(Cmp, DAG));
5309 ((K == LHS && K == TrueVal) || (K == RHS && K == FalseVal))) ||
5311 ((K == RHS && K == TrueVal) || (K == LHS && K == FalseVal)));
5332 EVT VT =
Op.getValueType();
5354 if (V1Tmp != TrueVal1 || V2Tmp != TrueVal2 || K1 != FalseVal1 ||
5367 int64_t PosVal = std::max(Val1, Val2);
5368 int64_t NegVal = std::min(Val1, Val2);
5415 V = (KTmp == TrueVal) ? FalseVal : TrueVal;
5420 if (*K != KTmp || V != VTmp)
5431bool ARMTargetLowering::isUnsupportedFloatingType(
EVT VT)
const {
5435 return !Subtarget->hasFP64();
5437 return !Subtarget->hasFullFP16();
5442 EVT VT =
Op.getValueType();
5446 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->
isThumb2())
5458 if (VT == MVT::i32 &&
5478 if (Subtarget->hasV8_1MMainlineOps() && CFVal && CTVal &&
5479 LHS.getValueType() == MVT::i32 && RHS.getValueType() == MVT::i32) {
5482 unsigned Opcode = 0;
5484 if (TVal == ~FVal) {
5486 }
else if (TVal == ~FVal + 1) {
5488 }
else if (TVal + 1 == FVal) {
5490 }
else if (TVal == FVal + 1) {
5522 return DAG.
getNode(Opcode, dl, VT, TrueVal, FalseVal, ARMcc, Cmp);
5526 if (isUnsupportedFloatingType(LHS.getValueType())) {
5528 DAG, LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS);
5532 if (!RHS.getNode()) {
5538 if (LHS.getValueType() == MVT::i32) {
5550 TrueVal.getValueType() == MVT::f32 ||
5551 TrueVal.getValueType() == MVT::f64)) {
5566 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5578 (
TrueVal.getValueType() == MVT::f16 ||
5579 TrueVal.getValueType() == MVT::f32 ||
5580 TrueVal.getValueType() == MVT::f64)) {
5581 bool swpCmpOps =
false;
5582 bool swpVselOps =
false;
5597 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5601 SDValue Cmp2 = getVFPCmp(LHS, RHS, DAG, dl);
5602 Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, CCR, Cmp2, DAG);
5612 if (!
N->hasOneUse())
5615 if (!
N->getNumValues())
5617 EVT VT =
Op.getValueType();
5618 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow())
5635 return DAG.
getLoad(MVT::i32,
SDLoc(
Op), Ld->getChain(), Ld->getBasePtr(),
5636 Ld->getPointerInfo(), Ld->getAlign(),
5637 Ld->getMemOperand()->getFlags());
5655 DAG.
getLoad(MVT::i32, dl, Ld->getChain(),
Ptr, Ld->getPointerInfo(),
5656 Ld->getAlign(), Ld->getMemOperand()->getFlags());
5658 EVT PtrType =
Ptr.getValueType();
5661 RetVal2 = DAG.
getLoad(MVT::i32, dl, Ld->getChain(), NewPtr,
5662 Ld->getPointerInfo().getWithOffset(4),
5664 Ld->getMemOperand()->getFlags());
5682 bool LHSSeenZero =
false;
5684 bool RHSSeenZero =
false;
5686 if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) {
5697 if (LHS.getValueType() == MVT::f32) {
5705 Chain, Dest, ARMcc, CCR, Cmp);
5717 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
5732 unsigned Opc =
Cond.getOpcode();
5735 if (
Cond.getResNo() == 1 &&
5745 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Cond, DAG, ARMcc);
5769 if (isUnsupportedFloatingType(LHS.getValueType())) {
5771 DAG, LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS);
5775 if (!RHS.getNode()) {
5783 unsigned Opc = LHS.getOpcode();
5797 std::tie(
Value, OverflowCmp) = getARMXALUOOp(LHS.getValue(0), DAG, ARMcc);
5812 if (LHS.getValueType() == MVT::i32) {
5817 Chain, Dest, ARMcc, CCR, Cmp);
5823 if (
SDValue Result = OptimizeVFPBrcond(
Op, DAG))
5834 SDValue Ops[] = { Chain, Dest, ARMcc, CCR,
Cmp };
5856 if (Subtarget->
isThumb2() || (Subtarget->hasV8MBaselineOps() && Subtarget->isThumb())) {
5862 Addr,
Op.getOperand(2), JTI);
5868 Chain =
Addr.getValue(1);
5875 Chain =
Addr.getValue(1);
5881 EVT VT =
Op.getValueType();
5884 if (
Op.getValueType().getVectorElementType() == MVT::i32) {
5885 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::f32)
5893 const EVT OpTy =
Op.getOperand(0).getValueType();
5894 if (OpTy == MVT::v4f32)
5896 else if (OpTy == MVT::v4f16 && HasFullFP16)
5898 else if (OpTy == MVT::v8f16 && HasFullFP16)
5903 if (VT != MVT::v4i16 && VT != MVT::v8i16)
5906 Op = DAG.
getNode(
Op.getOpcode(), dl, NewTy,
Op.getOperand(0));
5911 EVT VT =
Op.getValueType();
5915 bool IsStrict =
Op->isStrictFPOpcode();
5916 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
5918 if (isUnsupportedFloatingType(SrcVal.
getValueType())) {
5928 MakeLibCallOptions CallOptions;
5931 std::tie(Result, Chain) =
makeLibCall(DAG, LC,
Op.getValueType(), SrcVal,
5932 CallOptions, Loc, Chain);
5942 Loc,
Op.getValueType(), SrcVal);
5951 EVT VT =
Op.getValueType();
5953 EVT FromVT =
Op.getOperand(0).getValueType();
5955 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f32)
5957 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f64 &&
5958 Subtarget->hasFP64())
5960 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f16 &&
5961 Subtarget->hasFullFP16())
5963 if (VT == MVT::v4i32 && ToVT == MVT::i32 && FromVT == MVT::v4f32 &&
5964 Subtarget->hasMVEFloatOps())
5966 if (VT == MVT::v8i16 && ToVT == MVT::i16 && FromVT == MVT::v8f16 &&
5967 Subtarget->hasMVEFloatOps())
5970 if (FromVT != MVT::v4f32 && FromVT != MVT::v8f16)
5987 EVT VT =
Op.getValueType();
5990 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i32) {
5996 assert((
Op.getOperand(0).getValueType() == MVT::v4i16 ||
5997 Op.getOperand(0).getValueType() == MVT::v8i16) &&
5998 "Invalid type for custom lowering!");
6003 if (VT == MVT::v4f32)
6004 DestVecType = MVT::v4i32;
6005 else if (VT == MVT::v4f16 && HasFullFP16)
6006 DestVecType = MVT::v4i16;
6007 else if (VT == MVT::v8f16 && HasFullFP16)
6008 DestVecType = MVT::v8i16;
6014 switch (
Op.getOpcode()) {
6026 Op = DAG.
getNode(CastOpc, dl, DestVecType,
Op.getOperand(0));
6031 EVT VT =
Op.getValueType();
6034 if (isUnsupportedFloatingType(VT)) {
6042 MakeLibCallOptions CallOptions;
6044 CallOptions,
SDLoc(
Op)).first;
6055 EVT VT =
Op.getValueType();
6057 bool InGPR = Tmp0.
getOpcode() == ISD::BITCAST ||
6059 bool UseNEON = !InGPR && Subtarget->hasNEON();
6066 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 :
MVT::v1i64;
6069 DAG.
getNode(ISD::BITCAST, dl, OpVT, Mask),
6073 if (SrcVT == MVT::f32) {
6077 DAG.
getNode(ISD::BITCAST, dl, OpVT, Tmp1),
6079 }
else if (VT == MVT::f32)
6081 DAG.
getNode(ISD::BITCAST, dl, MVT::v1i64, Tmp1),
6083 Tmp0 = DAG.
getNode(ISD::BITCAST, dl, OpVT, Tmp0);
6084 Tmp1 = DAG.
getNode(ISD::BITCAST, dl, OpVT, Tmp1);
6095 if (VT == MVT::f32) {
6096 Res = DAG.
getNode(ISD::BITCAST, dl, MVT::v2f32, Res);
6100 Res = DAG.
getNode(ISD::BITCAST, dl, MVT::f64, Res);
6107 if (SrcVT == MVT::f64)
6110 Tmp1 = DAG.
getNode(ISD::BITCAST, dl, MVT::i32, Tmp1);
6116 if (VT == MVT::f32) {
6118 DAG.
getNode(ISD::BITCAST, dl, MVT::i32, Tmp0), Mask2);
6119 return DAG.
getNode(ISD::BITCAST, dl, MVT::f32,
6140 EVT VT =
Op.getValueType();
6142 unsigned Depth =
Op.getConstantOperandVal(0);
6144 SDValue FrameAddr = LowerFRAMEADDR(
Op, DAG);
6163 EVT VT =
Op.getValueType();
6165 unsigned Depth =
Op.getConstantOperandVal(0);
6179 .
Case(
"sp", ARM::SP)
6194 assert(
N->getValueType(0) == MVT::i64
6195 &&
"ExpandREAD_REGISTER called for non-i64 type result.");
6198 DAG.
getVTList(MVT::i32, MVT::i32, MVT::Other),
6238 const APInt &APIntIndex =
Index->getAPIntValue();
6240 NewIndex *= APIntIndex;
6252 SDValue BitCast = DAG.
getNode(ISD::BITCAST, dl, VecVT, ExtractSrc);
6270 EVT SrcVT =
Op.getValueType();
6271 EVT DstVT =
N->getValueType(0);
6273 if ((SrcVT == MVT::i16 || SrcVT == MVT::i32) &&
6274 (DstVT == MVT::f16 || DstVT == MVT::bf16))
6278 if ((DstVT == MVT::i16 || DstVT == MVT::i32) &&
6279 (SrcVT == MVT::f16 || SrcVT == MVT::bf16))
6284 if (!(SrcVT == MVT::i64 || DstVT == MVT::i64))
6288 if (SrcVT == MVT::i64 && TLI.
isTypeLegal(DstVT)) {
6295 return DAG.
getNode(ISD::BITCAST, dl, DstVT,
6300 if (DstVT == MVT::i64 && TLI.
isTypeLegal(SrcVT)) {
6329 return DAG.
getNode(ISD::BITCAST, dl, VT, Vmov);
6337 EVT VT =
Op.getValueType();
6356 SDValue LoBigShift = DAG.
getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
6364 ? DAG.
getNode(Opc, dl, VT, ShOpHi,
6381 EVT VT =
Op.getValueType();
6424 DAG.
getConstant(Intrinsic::arm_get_fpscr, dl, MVT::i32)};
6476 Chain, DAG.
getConstant(Intrinsic::arm_set_fpscr,
DL, MVT::i32), FPSCR};
6504 Chain, DAG.
getConstant(Intrinsic::arm_set_fpscr,
DL, MVT::i32), FPSCR};
6534 EVT VT =
N->getValueType(0);
6535 if (VT.
isVector() && ST->hasNEON()) {
6544 if (ElemTy == MVT::i8) {
6552 if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) &&
6555 unsigned NumBits = ElemTy.getSizeInBits();
6567 if (ElemTy == MVT::i64) {
6580 if (!ST->hasV6T2Ops())
6589 EVT VT =
N->getValueType(0);
6592 assert(ST->hasNEON() &&
"Custom ctpop lowering requires NEON.");
6593 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
6594 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
6595 "Unexpected type for custom ctpop lowering");
6603 unsigned EltSize = 8;
6625 while (
Op.getOpcode() == ISD::BITCAST)
6626 Op =
Op.getOperand(0);
6628 APInt SplatBits, SplatUndef;
6629 unsigned SplatBitSize;
6632 !BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6634 SplatBitSize > ElementBits)
6645 assert(VT.
isVector() &&
"vector shift count is not a vector type");
6649 return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
6660 assert(VT.
isVector() &&
"vector shift count is not a vector type");
6665 return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
6666 if (Cnt >= -(isNarrow ? ElementBits / 2 : ElementBits) && Cnt <= -1) {
6675 EVT VT =
N->getValueType(0);
6697 "unexpected vector shift opcode");
6699 if (
isVShiftRImm(
N->getOperand(1), VT,
false,
false, Cnt)) {
6700 unsigned VShiftOpc =
6702 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0),
6708 EVT ShiftVT =
N->getOperand(1).getValueType();
6711 unsigned VShiftOpc =
6713 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0), NegatedCount);
6718 EVT VT =
N->getValueType(0);
6727 "Unknown shift to lower!");
6729 unsigned ShOpc =
N->getOpcode();
6730 if (ST->hasMVEIntegerOps()) {
6760 DAG.
SplitScalar(
N->getOperand(0), dl, MVT::i32, MVT::i32);
6774 if (ST->isThumb1Only())
6779 std::tie(
Lo,
Hi) = DAG.
SplitScalar(
N->getOperand(0), dl, MVT::i32, MVT::i32);
6795 bool Invert =
false;
6802 EVT VT =
Op.getValueType();
6810 assert(ST->hasMVEIntegerOps() &&
6811 "No hardware support for integer vector comparison!");
6813 if (
Op.getValueType().getVectorElementType() != MVT::i1)
6836 Merged = DAG.
getNode(ISD::BITCAST, dl, CmpVT, Merged);
6838 Merged = DAG.
getNOT(dl, Merged, CmpVT);
6848 switch (SetCCOpcode) {
6852 if (ST->hasMVEFloatOps()) {
6855 Invert =
true; [[fallthrough]];
6860 case ISD::SETLT: Swap =
true; [[fallthrough]];
6864 case ISD::SETLE: Swap =
true; [[fallthrough]];
6880 Result = DAG.
getNOT(dl, Result, VT);
6883 case ISD::SETUO: Invert =
true; [[fallthrough]];
6892 Result = DAG.
getNOT(dl, Result, VT);
6898 switch (SetCCOpcode) {
6901 if (ST->hasMVEIntegerOps()) {
6904 Invert =
true; [[fallthrough]];
6907 case ISD::SETLT: Swap =
true; [[fallthrough]];
6909 case ISD::SETLE: Swap =
true; [[fallthrough]];
6918 if (ST->hasNEON() && Opc ==
ARMCC::EQ) {
6926 if (AndOp.getNode() && AndOp.getOpcode() == ISD::BITCAST)
6929 if (AndOp.getNode() && AndOp.getOpcode() ==
ISD::AND) {
6930 Op0 = DAG.
getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(0));
6931 Op1 = DAG.
getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(1));
6934 Result = DAG.
getNOT(dl, Result, VT);
6968 Result = DAG.
getNOT(dl, Result, VT);
6980 assert(LHS.getSimpleValueType().isInteger() &&
"SETCCCARRY is integer only.");
7010 unsigned OpCmode, Imm;
7021 switch (SplatBitSize) {
7026 assert((SplatBits & ~0xff) == 0 &&
"one byte splat value is too big");
7029 VT = is128Bits ? MVT::v16i8 : MVT::v8i8;
7034 VT = is128Bits ? MVT::v8i16 : MVT::v4i16;
7035 if ((SplatBits & ~0xff) == 0) {
7041 if ((SplatBits & ~0xff00) == 0) {
7044 Imm = SplatBits >> 8;
7054 VT = is128Bits ? MVT::v4i32 : MVT::v2i32;
7055 if ((SplatBits & ~0xff) == 0) {
7061 if ((SplatBits & ~0xff00) == 0) {
7064 Imm = SplatBits >> 8;
7067 if ((SplatBits & ~0xff0000) == 0) {
7070 Imm = SplatBits >> 16;
7073 if ((SplatBits & ~0xff000000) == 0) {
7076 Imm = SplatBits >> 24;
7083 if ((SplatBits & ~0xffff) == 0 &&
7084 ((SplatBits | SplatUndef) & 0xff) == 0xff) {
7087 Imm = SplatBits >> 8;
7095 if ((SplatBits & ~0xffffff) == 0 &&
7096 ((SplatBits | SplatUndef) & 0xffff) == 0xffff) {
7099 Imm = SplatBits >> 16;
7115 unsigned ImmMask = 1;
7117 for (
int ByteNum = 0; ByteNum < 8; ++ByteNum) {
7118 if (((SplatBits | SplatUndef) & BitMask) == BitMask) {
7120 }
else if ((SplatBits & BitMask) != 0) {
7129 VT = is128Bits ? MVT::v2i64 : MVT::v1i64;
7143 EVT VT =
Op.getValueType();
7144 bool IsDouble = (VT == MVT::f64);
7150 if (
ST->genExecuteOnly()) {
7152 assert((!
ST->isThumb1Only() ||
ST->hasV8MBaselineOps()) &&
7153 "Unexpected architecture");
7176 if (!
ST->hasVFP3Base())
7181 if (IsDouble && !Subtarget->hasFP64())
7188 if (IsDouble || !
ST->useNEONForSinglePrecisionFP()) {
7206 if (!
ST->hasNEON() || (!IsDouble && !
ST->useNEONForSinglePrecisionFP()))
7215 if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32))
7226 return DAG.
getNode(ISD::BITCAST,
DL, MVT::f64, VecConstant);
7243 return DAG.
getNode(ISD::BITCAST,
DL, MVT::f64, VecConstant);
7269 unsigned ExpectedElt = Imm;
7270 for (
unsigned i = 1; i < NumElts; ++i) {
7274 if (ExpectedElt == NumElts)
7277 if (M[i] < 0)
continue;
7278 if (ExpectedElt !=
static_cast<unsigned>(M[i]))
7286 bool &ReverseVEXT,
unsigned &Imm) {
7288 ReverseVEXT =
false;
7299 unsigned ExpectedElt = Imm;
7300 for (
unsigned i = 1; i < NumElts; ++i) {
7304 if (ExpectedElt == NumElts * 2) {
7309 if (M[i] < 0)
continue;
7310 if (ExpectedElt !=
static_cast<unsigned>(M[i]))
7325 return VT == MVT::v8i8 && M.size() == 8;
7330 if (Mask.size() == Elements * 2)
7331 return Index / Elements;
7332 return Mask[
Index] == 0 ? 0 : 1;
7362 if (M.size() != NumElts && M.size() != NumElts*2)
7370 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7372 for (
unsigned j = 0; j < NumElts; j += 2) {
7373 if ((M[i+j] >= 0 && (
unsigned) M[i+j] != j + WhichResult) ||
7374 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] != j + NumElts + WhichResult))
7379 if (M.size() == NumElts*2)
7394 if (M.size() != NumElts && M.size() != NumElts*2)
7397 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7399 for (
unsigned j = 0; j < NumElts; j += 2) {
7400 if ((M[i+j] >= 0 && (
unsigned) M[i+j] != j + WhichResult) ||
7401 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] != j + WhichResult))
7406 if (M.size() == NumElts*2)
7426 if (M.size() != NumElts && M.size() != NumElts*2)
7429 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7431 for (
unsigned j = 0; j < NumElts; ++j) {
7432 if (M[i+j] >= 0 && (
unsigned) M[i+j] != 2 * j + WhichResult)
7437 if (M.size() == NumElts*2)
7456 if (M.size() != NumElts && M.size() != NumElts*2)
7459 unsigned Half = NumElts / 2;
7460 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7462 for (
unsigned j = 0; j < NumElts; j += Half) {
7463 unsigned Idx = WhichResult;
7464 for (
unsigned k = 0; k < Half; ++k) {
7465 int MIdx = M[i + j + k];
7466 if (MIdx >= 0 && (
unsigned) MIdx != Idx)
7473 if (M.size() == NumElts*2)
7497 if (M.size() != NumElts && M.size() != NumElts*2)
7500 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7502 unsigned Idx = WhichResult * NumElts / 2;
7503 for (
unsigned j = 0; j < NumElts; j += 2) {
7504 if ((M[i+j] >= 0 && (
unsigned) M[i+j] != Idx) ||
7505 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] != Idx + NumElts))
7511 if (M.size() == NumElts*2)
7530 if (M.size() != NumElts && M.size() != NumElts*2)
7533 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7535 unsigned Idx = WhichResult * NumElts / 2;
7536 for (
unsigned j = 0; j < NumElts; j += 2) {
7537 if ((M[i+j] >= 0 && (
unsigned) M[i+j] != Idx) ||
7538 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] != Idx))
7544 if (M.size() == NumElts*2)
7557 unsigned &WhichResult,
7560 if (
isVTRNMask(ShuffleMask, VT, WhichResult))
7562 if (
isVUZPMask(ShuffleMask, VT, WhichResult))
7564 if (
isVZIPMask(ShuffleMask, VT, WhichResult))
7582 if (NumElts != M.size())
7586 for (
unsigned i = 0; i != NumElts; ++i)
7587 if (M[i] >= 0 && M[i] != (
int) (NumElts - 1 - i))
7596 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7604 int Ofs = Top ? 1 : 0;
7605 int Upper = SingleSource ? 0 : NumElts;
7606 for (
int i = 0, e = NumElts / 2; i != e; ++i) {
7607 if (M[i] >= 0 && M[i] != (i * 2) + Ofs)
7609 if (M[i + e] >= 0 && M[i + e] != (i * 2) + Ofs +
Upper)
7618 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7627 unsigned Offset = Top ? 0 : 1;
7628 unsigned N = SingleSource ? 0 : NumElts;
7629 for (
unsigned i = 0; i < NumElts; i += 2) {
7630 if (M[i] >= 0 && M[i] != (
int)i)
7632 if (M[i + 1] >= 0 && M[i + 1] != (
int)(
N + i +
Offset))
7641 if (NumElts != M.size())
7649 unsigned Off0 = rev ? NumElts / 2 : 0;
7650 unsigned Off1 = rev ? 0 : NumElts / 2;
7651 for (
unsigned i = 0; i < NumElts; i += 2) {
7652 if (M[i] >= 0 && M[i] != (
int)(Off0 + i / 2))
7654 if (M[i + 1] >= 0 && M[i + 1] != (
int)(Off1 + i / 2))
7670 if (!ST->hasMVEFloatOps())
7675 if (VT != MVT::v8f16)
7696 for (
unsigned i = 1; i < 4; i++) {
7723 if (!ST->hasMVEFloatOps())
7728 if (VT != MVT::v4f32)
7744 for (
unsigned i = 1; i < 4; i++) {
7746 return Trunc.
getOpcode() == ISD::FP_EXTEND &&
7767 Val =
N->getAsZExtVal();
7769 if (ST->isThumb1Only()) {
7770 if (Val <= 255 || ~Val <= 255)
7782 EVT VT =
Op.getValueType();
7784 assert(ST->hasMVEIntegerOps() &&
"LowerBUILD_VECTOR_i1 called without MVE!");
7788 unsigned BitsPerBool;
7792 }
else if (NumElts == 4) {
7795 }
else if (NumElts == 8) {
7798 }
else if (NumElts == 16) {
7809 return U.get().isUndef() || U.get() == FirstOp;
7817 unsigned Bits32 = 0;
7818 for (
unsigned i = 0; i < NumElts; ++i) {
7822 bool BitSet = V.isUndef() ?
false : V->getAsZExtVal();
7824 Bits32 |= BoolMask << (i * BitsPerBool);
7830 for (
unsigned i = 0; i < NumElts; ++i) {
7843 if (!ST->hasMVEIntegerOps())
7847 EVT VT =
Op.getValueType();
7857 if (
N != 1 &&
N != 2 &&
N != 4 &&
N != 8)
7861 for (
unsigned I = 2;
I < NumElts;
I++) {
7877 switch (
N->getOpcode()) {
7886 return N->getOperand(1).getNode() ==
Op;
7888 switch (
N->getConstantOperandVal(0)) {
7889 case Intrinsic::arm_mve_add_predicated:
7890 case Intrinsic::arm_mve_mul_predicated:
7891 case Intrinsic::arm_mve_qadd_predicated:
7892 case Intrinsic::arm_mve_vhadd:
7893 case Intrinsic::arm_mve_hadd_predicated:
7894 case Intrinsic::arm_mve_vqdmulh:
7895 case Intrinsic::arm_mve_qdmulh_predicated:
7896 case Intrinsic::arm_mve_vqrdmulh:
7897 case Intrinsic::arm_mve_qrdmulh_predicated:
7898 case Intrinsic::arm_mve_vqdmull:
7899 case Intrinsic::arm_mve_vqdmull_predicated:
7901 case Intrinsic::arm_mve_sub_predicated:
7902 case Intrinsic::arm_mve_qsub_predicated:
7903 case Intrinsic::arm_mve_vhsub:
7904 case Intrinsic::arm_mve_hsub_predicated:
7905 return N->getOperand(2).getNode() ==
Op;
7920 EVT VT =
Op.getValueType();
7928 APInt SplatBits, SplatUndef;
7929 unsigned SplatBitSize;
7931 if (BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
7938 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32) &&
7940 [BVN](
const SDNode *U) { return IsQRMVEInstruction(U, BVN); })) {
7941 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7942 : SplatBitSize == 16 ? MVT::v8i16
7949 if ((
ST->hasNEON() && SplatBitSize <= 64) ||
7950 (
ST->hasMVEIntegerOps() && SplatBitSize <= 64)) {
7955 SplatBitSize, DAG, dl, VmovVT, VT,
VMOVModImm);
7963 uint64_t NegatedImm = (~SplatBits).getZExtValue();
7965 NegatedImm, SplatUndef.
getZExtValue(), SplatBitSize, DAG, dl, VmovVT,
7973 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) {
7983 if (
ST->hasMVEIntegerOps() &&
7984 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32)) {
7985 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7986 : SplatBitSize == 16 ? MVT::v8i16
8003 bool isOnlyLowElement =
true;
8004 bool usesOnlyOneValue =
true;
8005 bool hasDominantValue =
false;
8012 for (
unsigned i = 0; i < NumElts; ++i) {
8017 isOnlyLowElement =
false;
8021 unsigned &
Count = ValueCounts[V];
8024 if (++
Count > (NumElts / 2)) {
8025 hasDominantValue =
true;
8029 if (ValueCounts.
size() != 1)
8030 usesOnlyOneValue =
false;
8031 if (!
Value.getNode() && !ValueCounts.
empty())
8034 if (ValueCounts.
empty())
8046 if (hasDominantValue && EltSize <= 32) {
8062 if (VT !=
Value->getOperand(0).getValueType()) {
8075 if (!usesOnlyOneValue) {
8078 for (
unsigned I = 0;
I < NumElts; ++
I) {
8093 assert(FVT == MVT::f32 || FVT == MVT::f16);
8094 MVT IVT = (FVT == MVT::f32) ? MVT::i32 :
MVT::i16;
8095 for (
unsigned i = 0; i < NumElts; ++i)
8100 Val = LowerBUILD_VECTOR(Val, DAG, ST);
8102 return DAG.
getNode(ISD::BITCAST, dl, VT, Val);
8104 if (usesOnlyOneValue) {
8131 if (
ST->hasNEON() && VT.
is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) {
8151 if (EltSize >= 32) {
8157 for (
unsigned i = 0; i < NumElts; ++i)
8160 return DAG.
getNode(ISD::BITCAST, dl, VT, Val);
8171 for (
unsigned i = 0 ; i < NumElts; ++i) {
8190 EVT VT =
Op.getValueType();
8193 struct ShuffleSourceInfo {
8195 unsigned MinElt = std::numeric_limits<unsigned>::max();
8196 unsigned MaxElt = 0;
8206 int WindowScale = 1;
8208 ShuffleSourceInfo(
SDValue Vec) : Vec(Vec), ShuffleVec(Vec) {}
8216 for (
unsigned i = 0; i < NumElts; ++i) {
8233 if (Source == Sources.
end())
8237 unsigned EltNo = V.getConstantOperandVal(1);
8244 if (Sources.
size() > 2)
8250 for (
auto &Source : Sources) {
8251 EVT SrcEltTy =
Source.Vec.getValueType().getVectorElementType();
8252 if (SrcEltTy.
bitsLT(SmallestEltTy))
8253 SmallestEltTy = SrcEltTy;
8255 unsigned ResMultiplier =
8263 for (
auto &Src : Sources) {
8264 EVT SrcVT = Src.ShuffleVec.getValueType();
8268 if (SrcVTSize == VTSize)
8277 if (SrcVTSize < VTSize) {
8278 if (2 * SrcVTSize != VTSize)
8284 DAG.
getUNDEF(Src.ShuffleVec.getValueType()));
8288 if (SrcVTSize != 2 * VTSize)
8291 if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
8296 if (Src.MinElt >= NumSrcElts) {
8301 Src.WindowBase = -NumSrcElts;
8302 }
else if (Src.MaxElt < NumSrcElts) {
8319 Src.WindowBase = -Src.MinElt;
8326 for (
auto &Src : Sources) {
8327 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
8328 if (SrcEltTy == SmallestEltTy)
8333 Src.WindowBase *= Src.WindowScale;
8339 assert(Src.ShuffleVec.getValueType() == ShuffleVT););
8346 if (Entry.isUndef())
8349 auto Src =
llvm::find(Sources, Entry.getOperand(0));
8358 int LanesDefined = BitsDefined / BitsPerShuffleLane;
8362 int *LaneMask = &
Mask[i * ResMultiplier];
8364 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
8365 ExtractBase += NumElts * (Src - Sources.begin());
8366 for (
int j = 0;
j < LanesDefined; ++
j)
8367 LaneMask[j] = ExtractBase + j;
8373 assert(Sources.size() <= 2 &&
"Too many sources!");
8376 for (
unsigned i = 0; i < Sources.size(); ++i)
8405 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8425 unsigned PFIndexes[4];
8426 for (
unsigned i = 0; i != 4; ++i) {
8430 PFIndexes[i] = M[i];
8434 unsigned PFTableIndex =
8435 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8437 unsigned Cost = (PFEntry >> 30);
8443 bool ReverseVEXT, isV_UNDEF;
8444 unsigned Imm, WhichResult;
8447 if (EltSize >= 32 ||
8454 else if (Subtarget->hasNEON() &&
8459 else if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8462 else if (Subtarget->hasMVEIntegerOps() &&
8466 else if (Subtarget->hasMVEIntegerOps() &&
8480 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8481 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8482 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8485 if (LHSID == (1*9+2)*9+3)
return LHS;
8486 assert(LHSID == ((4*9+5)*9+6)*9+7 &&
"Illegal OP_COPY!");
8543 for (
int I : ShuffleMask)
8546 if (V2.getNode()->isUndef())
8556 EVT VT =
Op.getValueType();
8558 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8559 "Expect an v8i16/v16i8 type");
8565 std::vector<int> NewMask;
8569 NewMask.push_back(i);
8609 if (VT != MVT::v16i1)
8620 return DAG.
getNode(ISD::BITCAST, dl, NewVT, PredAsVector);
8625 EVT VT =
Op.getValueType();
8629 assert(ST->hasMVEIntegerOps() &&
8630 "No support for vector shuffle of boolean predicates");
8656 "Expected identical vector type in expanded i1 shuffle!");
8660 PredAsVector2, ShuffleMask);
8665 if (VT == MVT::v2i1) {
8682 EVT VT =
Op.getValueType();
8686 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8687 "Unexpected vector type");
8689 int QuarterSize = NumElts / 4;
8698 for (
int i = 0; i <
Length; i++) {
8699 if (ShuffleMask[Start + i] >= 0) {
8700 if (ShuffleMask[Start + i] %
Length != i)
8702 MovIdx = ShuffleMask[Start + i] /
Length;
8710 for (
int i = 1; i <
Length; i++) {
8711 if (ShuffleMask[Start + i] >= 0 &&
8712 (ShuffleMask[Start + i] /
Length != MovIdx ||
8713 ShuffleMask[Start + i] %
Length != i))
8719 for (
int Part = 0; Part < 4; ++Part) {
8721 int Elt = getMovIdx(ShuffleMask, Part * QuarterSize, QuarterSize);
8725 Input =
Op->getOperand(1);
8735 if (!Parts[0] && !Parts[1] && !Parts[2] && !Parts[3])
8740 if (!Parts[0] || !Parts[1] || !Parts[2] || !Parts[3]) {
8742 for (
int Part = 0; Part < 4; ++Part)
8743 for (
int i = 0; i < QuarterSize; i++)
8745 Parts[Part] ? -1 : ShuffleMask[Part * QuarterSize + i]);
8747 VT, dl,
Op->getOperand(0),
Op->getOperand(1), NewShuffleMask);
8750 for (
int Part = 0; Part < 4; ++Part)
8766 EVT VT =
Op.getValueType();
8778 for (
int i = 0, NumMaskElts = Mask.size(); i < NumMaskElts; ++i) {
8782 if (Mask[i] != i + BaseOffset) {
8783 if (OffElement == -1)
8789 return NonUndef > 2 && OffElement != -1;
8793 if (isOneOffIdentityMask(ShuffleMask, VT, 0, OffElement))
8795 else if (isOneOffIdentityMask(ShuffleMask, VT, NumElts, OffElement))
8806 ShuffleMask[OffElement] < (
int)NumElts ? V1 : V2,
8817 EVT VT =
Op.getValueType();
8821 if (ST->hasMVEIntegerOps() && EltSize == 1)
8832 if (EltSize <= 32) {
8836 if (Lane == -1) Lane = 0;
8847 bool IsScalarToVector =
true;
8850 IsScalarToVector =
false;
8853 if (IsScalarToVector)
8860 bool ReverseVEXT =
false;
8862 if (ST->hasNEON() &&
isVEXTMask(ShuffleMask, VT, ReverseVEXT, Imm)) {
8886 unsigned WhichResult = 0;
8887 bool isV_UNDEF =
false;
8888 if (ST->hasNEON()) {
8890 ShuffleMask, VT, WhichResult, isV_UNDEF)) {
8897 if (ST->hasMVEIntegerOps()) {
8932 }) &&
"Unexpected shuffle index into UNDEF operand!");
8935 ShuffleMask, SubVT, WhichResult, isV_UNDEF)) {
8938 assert((WhichResult == 0) &&
8939 "In-place shuffle of concat can only have one result!");
8948 if (ST->hasMVEIntegerOps() && EltSize <= 32) {
8952 for (
bool Top : {
false,
true}) {
8953 for (
bool SingleSource : {
false,
true}) {
8954 if (
isTruncMask(ShuffleMask, VT, Top, SingleSource)) {
8959 SingleSource ? V1 : V2);
8975 unsigned PFIndexes[4];
8976 for (
unsigned i = 0; i != 4; ++i) {
8977 if (ShuffleMask[i] < 0)
8980 PFIndexes[i] = ShuffleMask[i];
8984 unsigned PFTableIndex =
8985 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8987 unsigned Cost = (PFEntry >> 30);
8993 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8994 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
9004 if (EltSize >= 32) {
9009 V1 = DAG.
getNode(ISD::BITCAST, dl, VecVT, V1);
9010 V2 = DAG.
getNode(ISD::BITCAST, dl, VecVT, V2);
9012 for (
unsigned i = 0; i < NumElts; ++i) {
9013 if (ShuffleMask[i] < 0)
9017 ShuffleMask[i] < (
int)NumElts ? V1 : V2,
9022 return DAG.
getNode(ISD::BITCAST, dl, VT, Val);
9025 if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
9029 if (ST->hasNEON() && VT == MVT::v8i8)
9033 if (ST->hasMVEIntegerOps())
9042 EVT VecVT =
Op.getOperand(0).getValueType();
9045 assert(ST->hasMVEIntegerOps() &&
9046 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
9050 unsigned Lane =
Op.getConstantOperandVal(2);
9051 unsigned LaneWidth =
9053 unsigned Mask = ((1 << LaneWidth) - 1) << Lane * LaneWidth;
9071 if (Subtarget->hasMVEIntegerOps() &&
9072 Op.getValueType().getScalarSizeInBits() == 1)
9096 IVecIn, IElt, Lane);
9097 return DAG.
getNode(ISD::BITCAST, dl, VecVT, IVecOut);
9105 EVT VecVT =
Op.getOperand(0).getValueType();
9108 assert(ST->hasMVEIntegerOps() &&
9109 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
9113 unsigned Lane =
Op.getConstantOperandVal(1);
9114 unsigned LaneWidth =
9145 assert(
Op.getValueType().getScalarSizeInBits() == 1 &&
9146 "Unexpected custom CONCAT_VECTORS lowering");
9148 "Unexpected custom CONCAT_VECTORS lowering");
9149 assert(ST->hasMVEIntegerOps() &&
9150 "CONCAT_VECTORS lowering only supported for MVE");
9154 EVT Op2VT = V2.getValueType();
9155 assert(Op1VT == Op2VT &&
"Operand types don't match!");
9156 assert((Op1VT == MVT::v2i1 || Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) &&
9157 "Unexpected i1 concat operations!");
9170 if (Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) {
9184 auto ExtractInto = [&DAG, &dl](
SDValue NewV,
SDValue ConVec,
unsigned &j) {
9185 EVT NewVT = NewV.getValueType();
9186 EVT ConcatVT = ConVec.getValueType();
9187 unsigned ExtScale = 1;
9188 if (NewVT == MVT::v2f64) {
9202 ConVec = ExtractInto(NewV1, ConVec, j);
9203 ConVec = ExtractInto(NewV2, ConVec, j);
9213 while (ConcatOps.
size() > 1) {
9214 for (
unsigned I = 0, E = ConcatOps.
size();
I != E;
I += 2) {
9217 ConcatOps[
I / 2] = ConcatPair(V1, V2);
9221 return ConcatOps[0];
9226 EVT VT =
Op->getValueType(0);
9233 "unexpected CONCAT_VECTORS");
9240 DAG.
getNode(ISD::BITCAST, dl, MVT::f64, Op0),
9244 DAG.
getNode(ISD::BITCAST, dl, MVT::f64, Op1),
9246 return DAG.
getNode(ISD::BITCAST, dl,
Op.getValueType(), Val);
9254 EVT VT =
Op.getValueType();
9257 unsigned Index = V2->getAsZExtVal();
9260 "Unexpected custom EXTRACT_SUBVECTOR lowering");
9261 assert(ST->hasMVEIntegerOps() &&
9262 "EXTRACT_SUBVECTOR lowering only supported for MVE");
9272 EVT SubVT = MVT::v4i32;
9274 for (
unsigned i =
Index, j = 0; i < (
Index + NumElts); i++, j += 2) {
9289 for (
unsigned i =
Index, j = 0; i < (
Index + NumElts); i++, j++) {
9305 assert(ST->hasMVEIntegerOps() &&
"Expected MVE!");
9306 EVT VT =
N->getValueType(0);
9307 assert((VT == MVT::v16i1 || VT == MVT::v8i1 || VT == MVT::v4i1) &&
9308 "Expected a vector i1 type!");
9310 EVT FromVT =
Op.getValueType();
9321 if (!Subtarget->hasMVEIntegerOps())
9324 EVT ToVT =
N->getValueType(0);
9367 if (ToVT != MVT::v8i16 && ToVT != MVT::v16i8)
9369 EVT FromVT =
N->getOperand(0).getValueType();
9370 if (FromVT != MVT::v8i32 && FromVT != MVT::v16i16)
9381 if (!Subtarget->hasMVEIntegerOps())
9386 EVT ToVT =
N->getValueType(0);
9387 if (ToVT != MVT::v16i32 && ToVT != MVT::v8i32 && ToVT != MVT::v16i16)
9390 EVT FromVT =
Op.getValueType();
9391 if (FromVT != MVT::v8i16 && FromVT != MVT::v16i8)
9402 SDValue Ext1 = Ext.getValue(1);
9405 Ext = DAG.
getNode(
N->getOpcode(),
DL, MVT::v8i32, Ext);
9406 Ext1 = DAG.
getNode(
N->getOpcode(),
DL, MVT::v8i32, Ext1);
9418 EVT VT =
N->getValueType(0);
9419 if (VT == MVT::v2i64 &&
N->getOpcode() == ISD::BITCAST) {
9425 unsigned HiElt = 1 - LoElt;
9430 if (!Lo0 || !Hi0 || !Lo1 || !Hi1)
9446 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
9450 unsigned HalfSize = EltSize / 2;
9452 if (!
isIntN(HalfSize,
C->getSExtValue()))
9455 if (!
isUIntN(HalfSize,
C->getZExtValue()))
9494 switch (OrigSimpleTy) {
9510 unsigned ExtOpcode) {
9533 if (ExtendedTy == LD->getMemoryVT())
9534 return DAG.
getLoad(LD->getMemoryVT(),
SDLoc(LD), LD->getChain(),
9535 LD->getBasePtr(), LD->getPointerInfo(), LD->getAlign(),
9536 LD->getMemOperand()->getFlags());
9542 LD->getChain(), LD->getBasePtr(), LD->getPointerInfo(),
9543 LD->getMemoryVT(), LD->getAlign(),
9544 LD->getMemOperand()->getFlags());
9563 "Expected extending load");
9569 DAG.
getNode(Opcode,
SDLoc(newLoad), LD->getValueType(0), newLoad);
9577 if (
N->getOpcode() == ISD::BITCAST) {
9580 BVN->
getValueType(0) == MVT::v4i32 &&
"expected v4i32 BUILD_VECTOR");
9588 EVT VT =
N->getValueType(0);
9594 for (
unsigned i = 0; i != NumElts; ++i) {
9595 const APInt &CInt =
N->getConstantOperandAPInt(i);
9604 unsigned Opcode =
N->getOpcode();
9615 unsigned Opcode =
N->getOpcode();
9628 EVT VT =
Op.getValueType();
9630 "unexpected type for custom-lowering ISD::MUL");
9631 SDNode *N0 =
Op.getOperand(0).getNode();
9632 SDNode *N1 =
Op.getOperand(1).getNode();
9633 unsigned NewOpc = 0;
9637 if (isN0SExt && isN1SExt)
9642 if (isN0ZExt && isN1ZExt)
9644 else if (isN1SExt || isN1ZExt) {
9661 if (VT == MVT::v2i64)
9678 "unexpected types for extended operands to VMULL");
9679 return DAG.
getNode(NewOpc,
DL, VT, Op0, Op1);
9695 DAG.
getNode(ISD::BITCAST,
DL, Op1VT, N00), Op1),
9697 DAG.
getNode(ISD::BITCAST,
DL, Op1VT, N01), Op1));
9714 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9721 X = DAG.
getNode(ISD::BITCAST, dl, MVT::v4i32,
X);
9724 X = DAG.
getNode(ISD::BITCAST, dl, MVT::v4f32,
X);
9748 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9751 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9759 N0 = DAG.
getNode(ISD::BITCAST, dl, MVT::v4i32, N0);
9762 N0 = DAG.
getNode(ISD::BITCAST, dl, MVT::v4f32, N0);
9772 EVT VT =
Op.getValueType();
9773 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9774 "unexpected type for custom-lowering ISD::SDIV");
9781 if (VT == MVT::v8i8) {
9809 EVT VT =
Op.getValueType();
9810 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9811 "unexpected type for custom-lowering ISD::UDIV");
9818 if (VT == MVT::v8i8) {
9857 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9860 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9864 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9872 N0 = DAG.
getNode(ISD::BITCAST, dl, MVT::v4i32, N0);
9875 N0 = DAG.
getNode(ISD::BITCAST, dl, MVT::v4f32, N0);
9885 EVT VT =
N->getValueType(0);
9899 Op.getOperand(1), Carry);
9913 Op.getOperand(1), Carry);
9946 bool ShouldUseSRet = Subtarget->
isAPCS_ABI();
9948 if (ShouldUseSRet) {
9950 const uint64_t ByteSize =
DL.getTypeAllocSize(RetTy);
9951 const Align StackAlign =
DL.getPrefTypeAlign(RetTy);
9958 Entry.IsSExt =
false;
9959 Entry.IsZExt =
false;
9960 Entry.IsSRet =
true;
9961 Args.push_back(Entry);
9968 Entry.IsSExt =
false;
9969 Entry.IsZExt =
false;
9970 Args.push_back(Entry);
9973 (ArgVT == MVT::f64) ? RTLIB::SINCOS_STRET_F64 : RTLIB::SINCOS_STRET_F32;
9983 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
9986 return CallResult.first;
10005 EVT VT =
Op.getValueType();
10006 assert((VT == MVT::i32 || VT == MVT::i64) &&
10007 "unexpected type for custom lowering DIV");
10013 const char *
Name =
nullptr;
10015 Name = (VT == MVT::i32) ?
"__rt_sdiv" :
"__rt_sdiv64";
10017 Name = (VT == MVT::i32) ?
"__rt_udiv" :
"__rt_udiv64";
10023 for (
auto AI : {1, 0}) {
10025 Arg.Node =
Op.getOperand(AI);
10026 Arg.Ty = Arg.Node.getValueType().getTypeForEVT(*DAG.
getContext());
10027 Args.push_back(Arg);
10030 CallLoweringInfo CLI(DAG);
10034 ES, std::move(Args));
10044ARMTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
10052 const bool MinSize =
ST.hasMinSize();
10053 const bool HasDivide =
ST.isThumb() ?
ST.hasDivideInThumbMode()
10054 :
ST.hasDivideInARMMode();
10058 if (
N->getOperand(0).getValueType().isVector())
10063 if (!(MinSize && HasDivide))
10076 if (Divisor.
sgt(128))
10084 assert(
Op.getValueType() == MVT::i32 &&
10085 "unexpected type for custom lowering DIV");
10091 return LowerWindowsDIVLibCall(
Op, DAG,
Signed, DBZCHK);
10097 if (
N->getValueType(0) == MVT::i32)
10105void ARMTargetLowering::ExpandDIV_Windows(
10111 assert(
Op.getValueType() == MVT::i64 &&
10112 "unexpected type for custom lowering DIV");
10129 EVT MemVT = LD->getMemoryVT();
10130 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10131 MemVT == MVT::v16i1) &&
10132 "Expected a predicate type!");
10133 assert(MemVT ==
Op.getValueType());
10135 "Expected a non-extending load");
10136 assert(LD->isUnindexed() &&
"Expected a unindexed load");
10150 ISD::EXTLOAD, dl, MVT::i32, LD->getChain(), LD->getBasePtr(),
10152 LD->getMemOperand());
10159 if (MemVT != MVT::v16i1)
10168 EVT MemVT =
LD->getMemoryVT();
10169 assert(
LD->isUnindexed() &&
"Loads should be unindexed at this point.");
10171 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10177 {LD->getChain(), LD->getBasePtr()}, MemVT,
LD->getMemOperand());
10187 EVT MemVT = ST->getMemoryVT();
10188 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10189 MemVT == MVT::v16i1) &&
10190 "Expected a predicate type!");
10191 assert(MemVT == ST->getValue().getValueType());
10192 assert(!ST->isTruncatingStore() &&
"Expected a non-extending store");
10193 assert(ST->isUnindexed() &&
"Expected a unindexed store");
10198 SDValue Build = ST->getValue();
10199 if (MemVT != MVT::v16i1) {
10218 ST->getChain(), dl, GRP, ST->getBasePtr(),
10220 ST->getMemOperand());
10226 EVT MemVT = ST->getMemoryVT();
10227 assert(ST->isUnindexed() &&
"Stores should be unindexed at this point.");
10229 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10245 {ST->getChain(), Lo, Hi, ST->getBasePtr()},
10246 MemVT, ST->getMemOperand());
10247 }
else if (Subtarget->hasMVEIntegerOps() &&
10248 ((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10249 MemVT == MVT::v16i1))) {
10264 MVT VT =
Op.getSimpleValueType();
10266 SDValue PassThru =
N->getPassThru();
10277 VT, dl,
N->getChain(),
N->getBasePtr(),
N->getOffset(), Mask, ZeroVec,
10278 N->getMemoryVT(),
N->getMemOperand(),
N->getAddressingMode(),
10279 N->getExtensionType(),
N->isExpandingLoad());
10281 bool PassThruIsCastZero = (PassThru.
getOpcode() == ISD::BITCAST ||
10284 if (!PassThru.
isUndef() && !PassThruIsCastZero)
10291 if (!ST->hasMVEIntegerOps())
10295 unsigned BaseOpcode = 0;
10296 switch (
Op->getOpcode()) {
10298 case ISD::VECREDUCE_FADD: BaseOpcode =
ISD::FADD;
break;
10299 case ISD::VECREDUCE_FMUL: BaseOpcode =
ISD::FMUL;
break;
10300 case ISD::VECREDUCE_MUL: BaseOpcode =
ISD::MUL;
break;
10301 case ISD::VECREDUCE_AND: BaseOpcode =
ISD::AND;
break;
10302 case ISD::VECREDUCE_OR: BaseOpcode =
ISD::OR;
break;
10303 case ISD::VECREDUCE_XOR: BaseOpcode =
ISD::XOR;
break;
10304 case ISD::VECREDUCE_FMAX: BaseOpcode = ISD::FMAXNUM;
break;
10305 case ISD::VECREDUCE_FMIN: BaseOpcode = ISD::FMINNUM;
break;
10312 unsigned NumActiveLanes = NumElts;
10314 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10315 NumActiveLanes == 2) &&
10316 "Only expected a power 2 vector size");
10320 while (NumActiveLanes > 4) {
10323 Op0 = DAG.
getNode(BaseOpcode, dl, VT, Op0, Rev);
10324 NumActiveLanes /= 2;
10328 if (NumActiveLanes == 4) {
10338 SDValue Res0 = DAG.
getNode(BaseOpcode, dl, EltVT, Ext0, Ext1,
Op->getFlags());
10339 SDValue Res1 = DAG.
getNode(BaseOpcode, dl, EltVT, Ext2, Ext3,
Op->getFlags());
10340 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Res0, Res1,
Op->getFlags());
10346 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Ext0, Ext1,
Op->getFlags());
10350 if (EltVT !=
Op->getValueType(0))
10357 if (!ST->hasMVEFloatOps())
10364 if (!ST->hasNEON())
10372 unsigned PairwiseIntrinsic = 0;
10373 switch (
Op->getOpcode()) {
10376 case ISD::VECREDUCE_UMIN:
10377 PairwiseIntrinsic = Intrinsic::arm_neon_vpminu;
10379 case ISD::VECREDUCE_UMAX:
10380 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxu;
10382 case ISD::VECREDUCE_SMIN:
10383 PairwiseIntrinsic = Intrinsic::arm_neon_vpmins;
10385 case ISD::VECREDUCE_SMAX:
10386 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxs;
10392 unsigned NumActiveLanes = NumElts;
10394 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10395 NumActiveLanes == 2) &&
10396 "Only expected a power 2 vector size");
10402 VT =
Lo.getValueType();
10404 NumActiveLanes /= 2;
10408 while (NumActiveLanes > 1) {
10410 NumActiveLanes /= 2;
10417 if (EltVT !=
Op.getValueType()) {
10418 unsigned Extend = 0;
10419 switch (
Op->getOpcode()) {
10422 case ISD::VECREDUCE_UMIN:
10423 case ISD::VECREDUCE_UMAX:
10426 case ISD::VECREDUCE_SMIN:
10427 case ISD::VECREDUCE_SMAX:
10431 Res = DAG.
getNode(Extend, dl,
Op.getValueType(), Res);
10453 SDValue Ops[] = {
N->getOperand(0),
10463 DAG.
getVTList(MVT::i32, MVT::Other), Ops);
10476 const SDValue Ops[] = {RegClass, V0, SubReg0, V1, SubReg1};
10478 DAG.
getMachineNode(TargetOpcode::REG_SEQUENCE, dl, MVT::Untyped, Ops), 0);
10482 SDLoc dl(V.getNode());
10483 auto [VLo, VHi] = DAG.
SplitScalar(V, dl, MVT::i32, MVT::i32);
10493 assert(
N->getValueType(0) == MVT::i64 &&
10494 "AtomicCmpSwap on types less than 64 should be legal");
10503 ARM::CMP_SWAP_64,
SDLoc(
N),
10504 DAG.
getVTList(MVT::Untyped, MVT::Untyped, MVT::Other), Ops);
10523 EVT VT =
Op.getValueType();
10532 if (isUnsupportedFloatingType(LHS.getValueType())) {
10534 DAG, LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS, Chain, IsSignaling);
10535 if (!RHS.getNode()) {
10536 RHS = DAG.
getConstant(0, dl, LHS.getValueType());
10556 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
10557 SDValue Result = getCMOV(dl, VT, False, True, ARMcc, CCR, Cmp, DAG);
10559 ARMcc = DAG.
getConstant(CondCode2, dl, MVT::i32);
10560 Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
10561 Result = getCMOV(dl, VT, Result, True, ARMcc, CCR, Cmp, DAG);
10577 switch (
Op.getOpcode()) {
10586 case ISD::BRCOND:
return LowerBRCOND(
Op, DAG);
10587 case ISD::BR_CC:
return LowerBR_CC(
Op, DAG);
10588 case ISD::BR_JT:
return LowerBR_JT(
Op, DAG);
10609 case ISD::BITCAST:
return ExpandBITCAST(
Op.getNode(), DAG, Subtarget);
10613 case ISD::SREM:
return LowerREM(
Op.getNode(), DAG);
10614 case ISD::UREM:
return LowerREM(
Op.getNode(), DAG);
10634 case ISD::SET_ROUNDING:
return LowerSET_ROUNDING(
Op, DAG);
10635 case ISD::SET_FPMODE:
10636 return LowerSET_FPMODE(
Op, DAG);
10637 case ISD::RESET_FPMODE:
10638 return LowerRESET_FPMODE(
Op, DAG);
10642 return LowerDIV_Windows(
Op, DAG,
true);
10646 return LowerDIV_Windows(
Op, DAG,
false);
10653 return LowerSignedALUO(
Op, DAG);
10656 return LowerUnsignedALUO(
Op, DAG);
10668 case ISD::VECREDUCE_MUL:
10669 case ISD::VECREDUCE_AND:
10670 case ISD::VECREDUCE_OR:
10671 case ISD::VECREDUCE_XOR:
10673 case ISD::VECREDUCE_FADD:
10674 case ISD::VECREDUCE_FMUL:
10675 case ISD::VECREDUCE_FMIN:
10676 case ISD::VECREDUCE_FMAX:
10678 case ISD::VECREDUCE_UMIN:
10679 case ISD::VECREDUCE_UMAX:
10680 case ISD::VECREDUCE_SMIN:
10681 case ISD::VECREDUCE_SMAX:
10683 case ISD::ATOMIC_LOAD:
10685 case ISD::FSINCOS:
return LowerFSINCOS(
Op, DAG);
10688 case ISD::DYNAMIC_STACKALLOC:
10690 return LowerDYNAMIC_STACKALLOC(
Op, DAG);
10695 case ISD::FP_EXTEND:
return LowerFP_EXTEND(
Op, DAG);
10699 return LowerSPONENTRY(
Op, DAG);
10706 unsigned IntNo =
N->getConstantOperandVal(0);
10708 if (IntNo == Intrinsic::arm_smlald)
10710 else if (IntNo == Intrinsic::arm_smlaldx)
10712 else if (IntNo == Intrinsic::arm_smlsld)
10714 else if (IntNo == Intrinsic::arm_smlsldx)
10721 std::tie(
Lo,
Hi) = DAG.
SplitScalar(
N->getOperand(3), dl, MVT::i32, MVT::i32);
10725 N->getOperand(1),
N->getOperand(2),
10737 switch (
N->getOpcode()) {
10744 Res = ExpandBITCAST(
N, DAG, Subtarget);
10753 Res = LowerREM(
N, DAG);
10757 Res = LowerDivRem(
SDValue(
N, 0), DAG);
10768 case ISD::READCYCLECOUNTER:
10776 case ISD::ATOMIC_CMP_SWAP:
10811 "ROPI/RWPI not currently supported with SjLj");
10820 bool isThumb = Subtarget->isThumb();
10821 bool isThumb2 = Subtarget->
isThumb2();
10824 unsigned PCAdj = (
isThumb || isThumb2) ? 4 : 8;
10830 : &ARM::GPRRegClass;
10848 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10854 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10860 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
10878 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10883 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10888 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
10893 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
10899 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
10914 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10920 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10944 : &ARM::GPRnopcRegClass;
10949 unsigned MaxCSNum = 0;
10957 if (!
II.isEHLabel())
10961 if (!MF->hasCallSiteLandingPad(
Sym))
continue;
10964 for (
unsigned Idx : CallSiteIdxs) {
10965 CallSiteNumToLPad[Idx].push_back(&BB);
10966 MaxCSNum = std::max(MaxCSNum, Idx);
10973 std::vector<MachineBasicBlock*> LPadList;
10975 LPadList.reserve(CallSiteNumToLPad.
size());
10976 for (
unsigned I = 1;
I <= MaxCSNum; ++
I) {
10979 LPadList.push_back(
MBB);
10984 assert(!LPadList.empty() &&
10985 "No landing pad destinations for the dispatch jump table!");
10999 unsigned trap_opcode;
11000 if (Subtarget->isThumb())
11001 trap_opcode = ARM::tTRAP;
11003 trap_opcode = Subtarget->useNaClTrap() ? ARM::TRAPNaCl : ARM::TRAP;
11012 MF->insert(MF->end(), DispatchBB);
11013 MF->insert(MF->end(), DispContBB);
11014 MF->insert(MF->end(), TrapBB);
11018 SetupEntryBlockForSjLj(
MI,
MBB, DispatchBB, FI);
11025 MIB =
BuildMI(DispatchBB, dl,
TII->get(ARM::Int_eh_sjlj_dispatchsetup));
11037 unsigned NumLPads = LPadList.size();
11039 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11040 BuildMI(DispatchBB, dl,
TII->get(ARM::t2LDRi12), NewVReg1)
11046 if (NumLPads < 256) {
11047 BuildMI(DispatchBB, dl,
TII->get(ARM::t2CMPri))
11049 .
addImm(LPadList.size())
11052 Register VReg1 =
MRI->createVirtualRegister(TRC);
11053 BuildMI(DispatchBB, dl,
TII->get(ARM::t2MOVi16), VReg1)
11054 .
addImm(NumLPads & 0xFFFF)
11057 unsigned VReg2 = VReg1;
11058 if ((NumLPads & 0xFFFF0000) != 0) {
11059 VReg2 =
MRI->createVirtualRegister(TRC);
11060 BuildMI(DispatchBB, dl,
TII->get(ARM::t2MOVTi16), VReg2)
11066 BuildMI(DispatchBB, dl,
TII->get(ARM::t2CMPrr))
11072 BuildMI(DispatchBB, dl,
TII->get(ARM::t2Bcc))
11077 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11078 BuildMI(DispContBB, dl,
TII->get(ARM::t2LEApcrelJT), NewVReg3)
11082 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11083 BuildMI(DispContBB, dl,
TII->get(ARM::t2ADDrs), NewVReg4)
11090 BuildMI(DispContBB, dl,
TII->get(ARM::t2BR_JT))
11094 }
else if (Subtarget->isThumb()) {
11095 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11096 BuildMI(DispatchBB, dl,
TII->get(ARM::tLDRspi), NewVReg1)
11102 if (NumLPads < 256) {
11103 BuildMI(DispatchBB, dl,
TII->get(ARM::tCMPi8))
11113 Align Alignment = MF->getDataLayout().getPrefTypeAlign(
Int32Ty);
11114 unsigned Idx =
ConstantPool->getConstantPoolIndex(
C, Alignment);
11116 Register VReg1 =
MRI->createVirtualRegister(TRC);
11117 BuildMI(DispatchBB, dl,
TII->get(ARM::tLDRpci))
11121 BuildMI(DispatchBB, dl,
TII->get(ARM::tCMPr))
11127 BuildMI(DispatchBB, dl,
TII->get(ARM::tBcc))
11132 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
11133 BuildMI(DispContBB, dl,
TII->get(ARM::tLSLri), NewVReg2)
11139 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11140 BuildMI(DispContBB, dl,
TII->get(ARM::tLEApcrelJT), NewVReg3)
11144 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11145 BuildMI(DispContBB, dl,
TII->get(ARM::tADDrr), NewVReg4)
11155 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
11156 BuildMI(DispContBB, dl,
TII->get(ARM::tLDRi), NewVReg5)
11162 unsigned NewVReg6 = NewVReg5;
11163 if (IsPositionIndependent) {
11164 NewVReg6 =
MRI->createVirtualRegister(TRC);
11165 BuildMI(DispContBB, dl,
TII->get(ARM::tADDrr), NewVReg6)
11172 BuildMI(DispContBB, dl,
TII->get(ARM::tBR_JTr))
11176 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11177 BuildMI(DispatchBB, dl,
TII->get(ARM::LDRi12), NewVReg1)
11183 if (NumLPads < 256) {
11184 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPri))
11188 }
else if (Subtarget->hasV6T2Ops() &&
isUInt<16>(NumLPads)) {
11189 Register VReg1 =
MRI->createVirtualRegister(TRC);
11190 BuildMI(DispatchBB, dl,
TII->get(ARM::MOVi16), VReg1)
11191 .
addImm(NumLPads & 0xFFFF)
11194 unsigned VReg2 = VReg1;
11195 if ((NumLPads & 0xFFFF0000) != 0) {
11196 VReg2 =
MRI->createVirtualRegister(TRC);
11197 BuildMI(DispatchBB, dl,
TII->get(ARM::MOVTi16), VReg2)
11203 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPrr))
11213 Align Alignment = MF->getDataLayout().getPrefTypeAlign(
Int32Ty);
11214 unsigned Idx =
ConstantPool->getConstantPoolIndex(
C, Alignment);
11216 Register VReg1 =
MRI->createVirtualRegister(TRC);
11217 BuildMI(DispatchBB, dl,
TII->get(ARM::LDRcp))
11222 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPrr))
11233 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11234 BuildMI(DispContBB, dl,
TII->get(ARM::MOVsi), NewVReg3)
11239 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11240 BuildMI(DispContBB, dl,
TII->get(ARM::LEApcrelJT), NewVReg4)
11247 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
11248 BuildMI(DispContBB, dl,
TII->get(ARM::LDRrs), NewVReg5)
11255 if (IsPositionIndependent) {
11256 BuildMI(DispContBB, dl,
TII->get(ARM::BR_JTadd))
11261 BuildMI(DispContBB, dl,
TII->get(ARM::BR_JTr))
11270 if (SeenMBBs.
insert(CurMBB).second)
11282 while (!Successors.empty()) {
11285 BB->removeSuccessor(SMBB);
11291 BB->normalizeSuccProbs();
11298 II = BB->rbegin(), IE = BB->rend();
II != IE; ++
II) {
11299 if (!
II->isCall())
continue;
11303 OI =
II->operands_begin(), OE =
II->operands_end();
11305 if (!OI->isReg())
continue;
11306 DefRegs[OI->getReg()] =
true;
11311 for (
unsigned i = 0; SavedRegs[i] != 0; ++i) {
11312 unsigned Reg = SavedRegs[i];
11314 !ARM::tGPRRegClass.contains(Reg) &&
11315 !ARM::hGPRRegClass.contains(Reg))
11317 if (Subtarget->
isThumb1Only() && !ARM::tGPRRegClass.contains(Reg))
11319 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg))
11332 MBBLPad->setIsEHPad(
false);
11335 MI.eraseFromParent();
11348static unsigned getLdOpcode(
unsigned LdSize,
bool IsThumb1,
bool IsThumb2) {
11350 return LdSize == 16 ? ARM::VLD1q32wb_fixed
11351 : LdSize == 8 ? ARM::VLD1d32wb_fixed : 0;
11353 return LdSize == 4 ? ARM::tLDRi
11354 : LdSize == 2 ? ARM::tLDRHi
11355 : LdSize == 1 ? ARM::tLDRBi : 0;
11357 return LdSize == 4 ? ARM::t2LDR_POST
11358 : LdSize == 2 ? ARM::t2LDRH_POST
11359 : LdSize == 1 ? ARM::t2LDRB_POST : 0;
11360 return LdSize == 4 ? ARM::LDR_POST_IMM
11361 : LdSize == 2 ? ARM::LDRH_POST
11362 : LdSize == 1 ? ARM::LDRB_POST_IMM : 0;
11367static unsigned getStOpcode(
unsigned StSize,
bool IsThumb1,
bool IsThumb2) {
11369 return StSize == 16 ? ARM::VST1q32wb_fixed
11370 : StSize == 8 ? ARM::VST1d32wb_fixed : 0;
11372 return StSize == 4 ? ARM::tSTRi
11373 : StSize == 2 ? ARM::tSTRHi
11374 : StSize == 1 ? ARM::tSTRBi : 0;
11376 return StSize == 4 ? ARM::t2STR_POST
11377 : StSize == 2 ? ARM::t2STRH_POST
11378 : StSize == 1 ? ARM::t2STRB_POST : 0;
11379 return StSize == 4 ? ARM::STR_POST_IMM
11380 : StSize == 2 ? ARM::STRH_POST
11381 : StSize == 1 ? ARM::STRB_POST_IMM : 0;
11388 unsigned LdSize,
unsigned Data,
unsigned AddrIn,
11389 unsigned AddrOut,
bool IsThumb1,
bool IsThumb2) {
11390 unsigned LdOpc =
getLdOpcode(LdSize, IsThumb1, IsThumb2);
11391 assert(LdOpc != 0 &&
"Should have a load opcode");
11398 }
else if (IsThumb1) {
11404 BuildMI(*BB, Pos, dl,
TII->get(ARM::tADDi8), AddrOut)
11409 }
else if (IsThumb2) {
11429 unsigned StSize,
unsigned Data,
unsigned AddrIn,
11430 unsigned AddrOut,
bool IsThumb1,
bool IsThumb2) {
11431 unsigned StOpc =
getStOpcode(StSize, IsThumb1, IsThumb2);
11432 assert(StOpc != 0 &&
"Should have a store opcode");
11434 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11439 }
else if (IsThumb1) {
11446 BuildMI(*BB, Pos, dl,
TII->get(ARM::tADDi8), AddrOut)
11451 }
else if (IsThumb2) {
11452 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11458 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11479 unsigned SizeVal =
MI.getOperand(2).getImm();
11480 unsigned Alignment =
MI.getOperand(3).getImm();
11485 unsigned UnitSize = 0;
11490 bool IsThumb2 = Subtarget->
isThumb2();
11491 bool IsThumb = Subtarget->isThumb();
11493 if (Alignment & 1) {
11495 }
else if (Alignment & 2) {
11500 Subtarget->hasNEON()) {
11501 if ((Alignment % 16 == 0) && SizeVal >= 16)
11503 else if ((Alignment % 8 == 0) && SizeVal >= 8)
11512 bool IsNeon = UnitSize >= 8;
11513 TRC = IsThumb ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
11515 VecTRC = UnitSize == 16 ? &ARM::DPairRegClass
11516 : UnitSize == 8 ? &ARM::DPRRegClass
11519 unsigned BytesLeft = SizeVal % UnitSize;
11520 unsigned LoopSize = SizeVal - BytesLeft;
11522 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) {
11526 unsigned srcIn = src;
11527 unsigned destIn = dest;
11528 for (
unsigned i = 0; i < LoopSize; i+=UnitSize) {
11529 Register srcOut =
MRI.createVirtualRegister(TRC);
11530 Register destOut =
MRI.createVirtualRegister(TRC);
11531 Register scratch =
MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11533 IsThumb1, IsThumb2);
11535 IsThumb1, IsThumb2);
11543 for (
unsigned i = 0; i < BytesLeft; i++) {
11544 Register srcOut =
MRI.createVirtualRegister(TRC);
11545 Register destOut =
MRI.createVirtualRegister(TRC);
11546 Register scratch =
MRI.createVirtualRegister(TRC);
11548 IsThumb1, IsThumb2);
11550 IsThumb1, IsThumb2);
11554 MI.eraseFromParent();
11580 MF->
insert(It, loopMBB);
11581 MF->
insert(It, exitMBB);
11584 unsigned CallFrameSize =
TII->getCallFrameSizeAt(
MI);
11594 Register varEnd =
MRI.createVirtualRegister(TRC);
11596 BuildMI(BB, dl,
TII->get(IsThumb ? ARM::t2MOVi32imm : ARM::MOVi32imm),
11599 }
else if (Subtarget->genExecuteOnly()) {
11600 assert(IsThumb &&
"Non-thumb expected to have used movt");
11609 unsigned Idx =
ConstantPool->getConstantPoolIndex(
C, Alignment);
11636 Register varLoop =
MRI.createVirtualRegister(TRC);
11637 Register varPhi =
MRI.createVirtualRegister(TRC);
11638 Register srcLoop =
MRI.createVirtualRegister(TRC);
11639 Register srcPhi =
MRI.createVirtualRegister(TRC);
11640 Register destLoop =
MRI.createVirtualRegister(TRC);
11641 Register destPhi =
MRI.createVirtualRegister(TRC);
11649 BuildMI(BB, dl,
TII->get(ARM::PHI), destPhi)
11655 Register scratch =
MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11657 IsThumb1, IsThumb2);
11659 IsThumb1, IsThumb2);
11663 BuildMI(*BB, BB->
end(), dl,
TII->get(ARM::tSUBi8), varLoop)
11671 TII->get(IsThumb2 ? ARM::t2SUBri : ARM::SUBri), varLoop);
11680 TII->get(IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc))
11689 auto StartOfExit = exitMBB->
begin();
11693 unsigned srcIn = srcLoop;
11694 unsigned destIn = destLoop;
11695 for (
unsigned i = 0; i < BytesLeft; i++) {
11696 Register srcOut =
MRI.createVirtualRegister(TRC);
11697 Register destOut =
MRI.createVirtualRegister(TRC);
11698 Register scratch =
MRI.createVirtualRegister(TRC);
11699 emitPostLd(BB, StartOfExit,
TII, dl, 1, scratch, srcIn, srcOut,
11700 IsThumb1, IsThumb2);
11701 emitPostSt(BB, StartOfExit,
TII, dl, 1, scratch, destIn, destOut,
11702 IsThumb1, IsThumb2);
11707 MI.eraseFromParent();
11719 "__chkstk is only supported on Windows");
11720 assert(Subtarget->
isThumb2() &&
"Windows on ARM requires Thumb-2 mode");
11740 switch (
TM.getCodeModel()) {
11758 Register Reg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11782 MI.eraseFromParent();
11806 .
addReg(
MI.getOperand(0).getReg())
11814 MI.eraseFromParent();
11838 if (miI == BB->
end()) {
11840 if (Succ->isLiveIn(ARM::CPSR))
11846 SelectItr->addRegisterKilled(ARM::CPSR,
TRI);
11858 Register AddDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11859 BuildMI(TpEntry, Dl,
TII->get(ARM::t2ADDri), AddDestReg)
11865 Register LsrDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11866 BuildMI(TpEntry, Dl,
TII->get(ARM::t2LSRri), LsrDestReg)
11872 Register TotalIterationsReg =
MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11873 BuildMI(TpEntry, Dl,
TII->get(ARM::t2WhileLoopSetup), TotalIterationsReg)
11876 BuildMI(TpEntry, Dl,
TII->get(ARM::t2WhileLoopStart))
11877 .
addUse(TotalIterationsReg)
11884 return TotalIterationsReg;
11895 Register TotalIterationsReg,
bool IsMemcpy) {
11902 SrcPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11903 CurrSrcReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11904 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), SrcPhiReg)
11912 Register DestPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11913 Register CurrDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11914 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), DestPhiReg)
11921 Register LoopCounterPhiReg =
MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11922 Register RemainingLoopIterationsReg =
11923 MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11924 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), LoopCounterPhiReg)
11925 .
addUse(TotalIterationsReg)
11927 .
addUse(RemainingLoopIterationsReg)
11931 Register PredCounterPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11932 Register RemainingElementsReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11933 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), PredCounterPhiReg)
11934 .
addUse(ElementCountReg)
11936 .
addUse(RemainingElementsReg)
11940 Register VccrReg =
MRI.createVirtualRegister(&ARM::VCCRRegClass);
11941 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VCTP8), VccrReg)
11942 .
addUse(PredCounterPhiReg)
11947 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2SUBri), RemainingElementsReg)
11948 .
addUse(PredCounterPhiReg)
11956 SrcValueReg =
MRI.createVirtualRegister(&ARM::MQPRRegClass);
11957 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VLDRBU8_post))
11966 SrcValueReg = OpSrcReg;
11968 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VSTRBU8_post))
11979 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2LoopDec), RemainingLoopIterationsReg)
11980 .
addUse(LoopCounterPhiReg)
11983 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2LoopEnd))
11984 .
addUse(RemainingLoopIterationsReg)
11997 bool isThumb2 = Subtarget->
isThumb2();
11998 switch (
MI.getOpcode()) {
12005 case ARM::tLDR_postidx: {
12009 .
add(
MI.getOperand(2))
12010 .
add(
MI.getOperand(3))
12011 .
add(
MI.getOperand(4))
12012 .
add(
MI.getOperand(0))
12014 MI.eraseFromParent();
12018 case ARM::MVE_MEMCPYLOOPINST:
12019 case ARM::MVE_MEMSETLOOPINST: {
12049 Register OpDestReg =
MI.getOperand(0).getReg();
12050 Register OpSrcReg =
MI.getOperand(1).getReg();
12051 Register OpSizeReg =
MI.getOperand(2).getReg();
12071 if (TpExit == BB) {
12073 "block containing memcpy/memset Pseudo");
12086 bool IsMemcpy =
MI.getOpcode() == ARM::MVE_MEMCPYLOOPINST;
12088 OpDestReg, OpSizeReg, TotalIterationsReg, IsMemcpy);
12103 MI.eraseFromParent();
12113 case ARM::t2STR_preidx:
12114 MI.setDesc(
TII->get(ARM::t2STR_PRE));
12116 case ARM::t2STRB_preidx:
12117 MI.setDesc(
TII->get(ARM::t2STRB_PRE));
12119 case ARM::t2STRH_preidx:
12120 MI.setDesc(
TII->get(ARM::t2STRH_PRE));
12123 case ARM::STRi_preidx:
12124 case ARM::STRBi_preidx: {
12125 unsigned NewOpc =
MI.getOpcode() == ARM::STRi_preidx ? ARM::STR_PRE_IMM
12126 : ARM::STRB_PRE_IMM;
12128 unsigned Offset =
MI.getOperand(4).getImm();
12136 .
add(
MI.getOperand(0))
12137 .
add(
MI.getOperand(1))
12138 .
add(
MI.getOperand(2))
12140 .
add(
MI.getOperand(5))
12141 .
add(
MI.getOperand(6))
12143 MI.eraseFromParent();
12146 case ARM::STRr_preidx:
12147 case ARM::STRBr_preidx:
12148 case ARM::STRH_preidx: {
12150 switch (
MI.getOpcode()) {
12152 case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG;
break;
12153 case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG;
break;
12154 case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE;
break;
12159 MI.eraseFromParent();
12163 case ARM::tMOVCCr_pseudo: {
12182 F->insert(It, sinkMBB);
12185 unsigned CallFrameSize =
TII->getCallFrameSizeAt(
MI);
12191 if (!
MI.killsRegister(ARM::CPSR,
nullptr) &&
12207 .
addImm(
MI.getOperand(3).getImm())
12208 .
addReg(
MI.getOperand(4).getReg());
12223 .
addReg(
MI.getOperand(1).getReg())
12225 .
addReg(
MI.getOperand(2).getReg())
12228 MI.eraseFromParent();
12233 case ARM::BCCZi64: {
12239 bool RHSisZero =
MI.getOpcode() == ARM::BCCZi64;
12244 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12248 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12254 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12258 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12268 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
12277 MI.eraseFromParent();
12281 case ARM::Int_eh_sjlj_setjmp:
12282 case ARM::Int_eh_sjlj_setjmp_nofp:
12283 case ARM::tInt_eh_sjlj_setjmp:
12284 case ARM::t2Int_eh_sjlj_setjmp:
12285 case ARM::t2Int_eh_sjlj_setjmp_nofp:
12288 case ARM::Int_eh_sjlj_setup_dispatch:
12289 EmitSjLjDispatchBlock(
MI, BB);
12312 Fn->
insert(BBI, SinkBB);
12314 Register ABSSrcReg =
MI.getOperand(1).getReg();
12315 Register ABSDstReg =
MI.getOperand(0).getReg();
12316 bool ABSSrcKIll =
MI.getOperand(1).isKill();
12317 bool isThumb2 = Subtarget->
isThumb2();
12321 Register NewRsbDstReg =
MRI.createVirtualRegister(
12322 isThumb2 ? &ARM::rGPRRegClass : &ARM::GPRRegClass);
12336 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12343 TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)).
addMBB(SinkBB)
12350 TII->get(isThumb2 ? ARM::t2RSBri : ARM::RSBri), NewRsbDstReg)
12359 TII->get(ARM::PHI), ABSDstReg)
12364 MI.eraseFromParent();
12369 case ARM::COPY_STRUCT_BYVAL_I32:
12371 return EmitStructByval(
MI, BB);
12372 case ARM::WIN__CHKSTK:
12373 return EmitLowered__chkstk(
MI, BB);
12374 case ARM::WIN__DBZCHK:
12375 return EmitLowered__dbzchk(
MI, BB);
12392 if (!
Node->hasAnyUseOfValue(0)) {
12393 MI.getOperand(0).setIsDead(
true);
12395 if (!
Node->hasAnyUseOfValue(1)) {
12396 MI.getOperand(1).setIsDead(
true);
12400 for (
unsigned I = 0;
I !=
MI.getOperand(4).
getImm(); ++
I) {
12401 Register TmpReg =
MRI.createVirtualRegister(isThumb1 ? &ARM::tGPRRegClass
12402 : &ARM::GPRRegClass);
12409 if (
MI.getOpcode() == ARM::MEMCPY) {
12427 MCID = &
TII->get(NewOpc);
12430 MI.getDesc().getNumOperands() + 5 -
MI.getDesc().getSize()
12431 &&
"converted opcode should be the same except for cc_out"
12432 " (and, on Thumb1, pred)");
12442 MI.addOperand(
MI.getOperand(1));
12443 MI.removeOperand(1);
12447 for (
unsigned i =
MI.getNumOperands(); i--;) {
12449 if (
op.isReg() &&
op.isUse()) {
12452 MI.tieOperands(DefIdx, i);
12466 if (!
MI.hasOptionalDef() || !MCID->
operands()[ccOutIdx].isOptionalDef()) {
12467 assert(!NewOpc &&
"Optional cc_out operand required");
12472 bool definesCPSR =
false;
12473 bool deadCPSR =
false;
12474 for (
unsigned i = MCID->
getNumOperands(), e =
MI.getNumOperands(); i != e;
12478 definesCPSR =
true;
12481 MI.removeOperand(i);
12485 if (!definesCPSR) {
12486 assert(!NewOpc &&
"Optional cc_out operand required");
12489 assert(deadCPSR == !Node->hasAnyUseOfValue(1) &&
"inconsistent dead flag");
12491 assert(!
MI.getOperand(ccOutIdx).getReg() &&
12492 "expect uninitialized optional cc_out operand");
12530 switch (
N->getOpcode()) {
12531 default:
return false;
12533 CC =
N->getOperand(0);
12555 EVT VT =
N->getValueType(0);
12556 CC =
N->getOperand(0);
12557 if (
CC.getValueType() != MVT::i1 ||
CC.getOpcode() !=
ISD::SETCC)
12603 EVT VT =
N->getValueType(0);
12606 bool SwapSelectOps;
12608 NonConstantVal, DAG))
12614 OtherOp, NonConstantVal);
12620 CCOp, TrueVal, FalseVal);
12644 if (
N->getOpcode() ==
ARMISD::VTRN &&
N->getValueType(0) == MVT::v2i32)
12659 if (!
N->getValueType(0).is64BitVector())
12667 EVT VT =
N->getValueType(0);
12706 EVT VT =
N->getValueType(0);
12712 Opcode = Intrinsic::arm_neon_vpaddls;
12714 Opcode = Intrinsic::arm_neon_vpaddlu;
12742 EVT VT =
N->getValueType(0);
12757 unsigned nextIndex = 0;
12825 return DAG.
getNode(ExtOp, dl, VT, tmp);
12856 if (SRA.getOpcode() !=
ISD::SRA) {
12863 if (Const->getZExtValue() != 31)
12868 if (SRA.getOperand(0) !=
Mul)
12872 SDLoc dl(AddcNode);
12873 unsigned Opcode = 0;
12901 SDValue HiMLALResult(SMLAL.getNode(), 1);
12902 SDValue LoMLALResult(SMLAL.getNode(), 0);
12908 SDValue resNode(AddcNode, 0);
12937 "Expect an ADDE or SUBE");
12941 "ADDE node has the wrong inputs");
12960 "Expect ADDC with two result values. First: i32");
12980 bool IsLeftOperandMUL =
false;
12985 IsLeftOperandMUL =
true;
12996 SDValue *LowAddSub =
nullptr;
12999 if ((AddeSubeOp0 != MULOp.
getValue(1)) && (AddeSubeOp1 != MULOp.
getValue(1)))
13002 if (IsLeftOperandMUL)
13003 HiAddSub = &AddeSubeOp1;
13005 HiAddSub = &AddeSubeOp0;
13010 if (AddcSubcOp0 == MULOp.
getValue(0)) {
13011 LoMul = &AddcSubcOp0;
13012 LowAddSub = &AddcSubcOp1;
13014 if (AddcSubcOp1 == MULOp.
getValue(0)) {
13015 LoMul = &AddcSubcOp1;
13016 LowAddSub = &AddcSubcOp0;
13024 if (AddcSubcNode == HiAddSub->getNode() ||
13040 if (Subtarget->hasV6Ops() && Subtarget->hasDSP() && Subtarget->
useMulOps() &&
13054 return SDValue(AddeSubeNode, 0);
13065 DAG.
getVTList(MVT::i32, MVT::i32), Ops);
13075 return SDValue(AddeSubeNode, 0);
13087 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
13096 SDNode *UmlalNode =
nullptr;
13121 DAG.
getVTList(MVT::i32, MVT::i32), Ops);
13135 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
13149 {N->getOperand(0), N->getOperand(1),
13150 AddcNode->getOperand(0), AddcNode->getOperand(1)});
13174 int32_t imm =
C->getSExtValue();
13175 if (imm < 0 && imm > std::numeric_limits<int>::min()) {
13180 return DAG.
getNode(Opcode,
DL,
N->getVTList(),
N->getOperand(0), RHS);
13195 int64_t imm =
C->getSExtValue();
13206 return DAG.
getNode(Opcode,
DL,
N->getVTList(),
13207 N->getOperand(0), RHS,
N->getOperand(2));
13219 if (!Subtarget->hasMVEIntegerOps())
13236 TrueVal =
N->getOperand(1);
13237 FalseVal =
N->getOperand(2);
13239 LHS =
N->getOperand(0);
13240 RHS =
N->getOperand(1);
13242 TrueVal =
N->getOperand(2);
13243 FalseVal =
N->getOperand(3);
13248 unsigned int Opcode = 0;
13249 if ((TrueVal->getOpcode() == ISD::VECREDUCE_UMIN ||
13250 FalseVal->getOpcode() == ISD::VECREDUCE_UMIN) &&
13255 }
else if ((TrueVal->getOpcode() == ISD::VECREDUCE_SMIN ||
13256 FalseVal->getOpcode() == ISD::VECREDUCE_SMIN) &&
13261 }
else if ((TrueVal->getOpcode() == ISD::VECREDUCE_UMAX ||
13262 FalseVal->getOpcode() == ISD::VECREDUCE_UMAX) &&
13267 }
else if ((TrueVal->getOpcode() == ISD::VECREDUCE_SMAX ||
13268 FalseVal->getOpcode() == ISD::VECREDUCE_SMAX) &&
13277 switch (TrueVal->getOpcode()) {
13278 case ISD::VECREDUCE_UMIN:
13279 case ISD::VECREDUCE_SMIN:
13280 case ISD::VECREDUCE_UMAX:
13281 case ISD::VECREDUCE_SMAX:
13296 if (TrueVal != LHS || FalseVal != RHS)
13299 EVT LeftType = LHS->getValueType(0);
13300 EVT RightType = RHS->getValueType(0);
13303 if (LeftType != VectorScalarType || RightType != VectorScalarType)
13307 if (VectorScalarType != MVT::i32)
13312 DCI.
DAG.
getNode(Opcode, dl, MVT::i32, LHS, RHS->getOperand(0));
13315 if (VectorScalarType != MVT::i32)
13328 EVT VT =
N->getValueType(0);
13343 Cmp.getOperand(0) !=
N->getOperand(1) ||
13344 Cmp.getOperand(1) !=
N->getOperand(2))
13358 ScalarType = MVT::i8;
13361 case (1 << 15) - 1:
13362 ScalarType = MVT::i16;
13365 case (1ULL << 31) - 1:
13366 ScalarType = MVT::i32;
13397 unsigned LegalLanes = 128 / (ShftAmt + 1);
13421 for (
unsigned I = 0;
I < NumParts; ++
I) {
13438 if (!Subtarget->hasMVEIntegerOps())
13453 if (
N->getOperand(0).getOpcode() !=
ISD::XOR)
13463 if (!Const || !Const->isOne())
13481 EVT VT =
N->getValueType(0);
13483 if (!Subtarget->hasMVEIntegerOps() ||
13512 Opc = Intrinsic::arm_mve_vctp64;
13515 Opc = Intrinsic::arm_mve_vctp32;
13518 Opc = Intrinsic::arm_mve_vctp16;
13521 Opc = Intrinsic::arm_mve_vctp8;
13575 EVT VT =
N->getValueType(0);
13581 switch (
Op.getOpcode()) {
13582 case ISD::VECREDUCE_ADD:
13606 unsigned N0RedOp = 0;
13613 unsigned N1RedOp = 0;
13627 if (
SDValue R = DistrubuteAddAddVecReduce(N0, N1))
13629 if (
SDValue R = DistrubuteAddAddVecReduce(N1, N0))
13636 auto DistrubuteVecReduceLoad = [&](
SDValue N0,
SDValue N1,
bool IsForward) {
13660 if (!BaseLocDecomp0.getBase() ||
13661 BaseLocDecomp0.getBase() != BaseLocDecomp1.getBase() ||
13662 !BaseLocDecomp0.hasValidOffset() || !BaseLocDecomp1.hasValidOffset())
13664 if (BaseLocDecomp0.getOffset() < BaseLocDecomp1.getOffset())
13666 if (BaseLocDecomp0.getOffset() > BaseLocDecomp1.getOffset())
13676 if (IsBefore < 0) {
13679 }
else if (IsBefore > 0) {
13692 }
else if (IsForward && IsVecReduce(N0) && IsVecReduce(N1) &&
13702 if (!IsVecReduce(N0) || !IsVecReduce(N1))
13712 if (
SDValue R = DistrubuteVecReduceLoad(N0, N1,
true))
13714 if (
SDValue R = DistrubuteVecReduceLoad(N1, N0,
false))
13721 if (!Subtarget->hasMVEIntegerOps())
13727 EVT VT =
N->getValueType(0);
13732 if (VT != MVT::i64)
13743 auto MakeVecReduce = [&](
unsigned Opcode,
unsigned OpcodeA,
SDValue NA,
13761 std::tie(Ops[0], Ops[1]) = DAG.
SplitScalar(NA, dl, MVT::i32, MVT::i32);
13763 unsigned S = VecRed->
getOpcode() == OpcodeA ? 2 : 0;
13812 "Expected shift op");
13831 if (Const->getAPIntValue().ult(256))
13834 Const->getAPIntValue().sgt(-256))
13850 (
N->getOperand(0).getOpcode() ==
ISD::SHL ||
13851 N->getOperand(0).getOpcode() ==
ISD::SRL) &&
13852 "Expected XOR(SHIFT) pattern");
13857 if (XorC && ShiftC) {
13858 unsigned MaskIdx, MaskLen;
13859 if (XorC->getAPIntValue().isShiftedMask(MaskIdx, MaskLen)) {
13860 unsigned ShiftAmt = ShiftC->getZExtValue();
13861 unsigned BitWidth =
N->getValueType(0).getScalarSizeInBits();
13862 if (
N->getOperand(0).getOpcode() ==
ISD::SHL)
13863 return MaskIdx == ShiftAmt && MaskLen == (
BitWidth - ShiftAmt);
13864 return MaskIdx == 0 && MaskLen == (
BitWidth - ShiftAmt);
13874 N->getOperand(0).getOpcode() ==
ISD::SRL) ||
13876 N->getOperand(0).getOpcode() ==
ISD::SHL)) &&
13877 "Expected shift-shift mask");
13890 return Subtarget->hasMVEIntegerOps() &&
isTypeLegal(VT);
13894 if (!Subtarget->hasNEON()) {
13913 return Subtarget->hasFP64();
13916 return Subtarget->hasMVEFloatOps();
13945 if (ST->isThumb() && ST->isThumb1Only())
13949 for (
auto *U :
N->uses()) {
13950 switch(U->getOpcode()) {
13968 if (U->getOperand(0).getOpcode() ==
ISD::SHL ||
13969 U->getOperand(1).getOpcode() ==
ISD::SHL)
13979 if (
N->getOperand(0).getOpcode() !=
ISD::SHL)
13986 if (!C1ShlC2 || !C2)
13989 APInt C2Int = C2->getAPIntValue();
13990 APInt C1Int = C1ShlC2->getAPIntValue();
13992 if (C2Int.
uge(C2Width))
13998 if ((C1Int & Mask) != C1Int)
14005 auto LargeImm = [](
const APInt &Imm) {
14006 unsigned Zeros = Imm.countl_zero() + Imm.countr_zero();
14007 return Imm.getBitWidth() - Zeros > 8;
14010 if (LargeImm(C1Int) || LargeImm(C2Int))
14022 SHL.dump();
N->dump());
14057 if (CSINC.getOpcode() !=
ARMISD::CSINC || !CSINC.hasOneUse())
14066 CSINC.getOperand(0)),
14067 CSINC.getOperand(1), CSINC.getOperand(2),
14068 CSINC.getOperand(3));
14087 if (!Subtarget->hasMVEIntegerOps() || !
N->getValueType(0).isVector())
14129 if (!Subtarget->hasVMLxForwarding())
14148 EVT VT =
N->getValueType(0);
14159 EVT VT =
N->getValueType(0);
14160 if (VT != MVT::v2i64)
14171 return Op->getOperand(0);
14184 if (
And->getOpcode() == ISD::BITCAST)
14185 And =
And->getOperand(0);
14189 if (Mask->getOpcode() == ISD::BITCAST)
14190 Mask = Mask->getOperand(0);
14193 Mask.getValueType() != MVT::v4i32)
14199 return And->getOperand(0);
14204 if (
SDValue Op0 = IsSignExt(N0)) {
14205 if (
SDValue Op1 = IsSignExt(N1)) {
14211 if (
SDValue Op0 = IsZeroExt(N0)) {
14212 if (
SDValue Op1 = IsZeroExt(N1)) {
14227 EVT VT =
N->getValueType(0);
14228 if (Subtarget->hasMVEIntegerOps() && VT == MVT::v2i64)
14239 if (VT != MVT::i32)
14246 int64_t MulAmt =
C->getSExtValue();
14249 ShiftAmt = ShiftAmt & (32 - 1);
14254 MulAmt >>= ShiftAmt;
14315 if (
N->getValueType(0) != MVT::i32)
14324 if (C1 == 255 || C1 == 65535)
14341 if (!C2 || C2 >= 32)
14385 if (Trailing == C2 && C2 + C3 < 32) {
14398 if (Leading == C2 && C2 + C3 < 32) {
14426 EVT VT =
N->getValueType(0);
14430 VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1)
14433 APInt SplatBits, SplatUndef;
14434 unsigned SplatBitSize;
14436 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14437 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14438 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14439 SplatBitSize == 64) {
14473 if (!Subtarget->hasV6Ops() ||
14474 (Subtarget->isThumb() &&
14475 (!Subtarget->hasThumb2() || !Subtarget->hasDSP())))
14478 SDValue SRL = OR->getOperand(0);
14479 SDValue SHL = OR->getOperand(1);
14482 SRL = OR->getOperand(1);
14483 SHL = OR->getOperand(0);
14490 if ((SRL.getOperand(0).getNode() != SHL.getOperand(0).getNode()) ||
14494 SDNode *SMULLOHI = SRL.getOperand(0).getNode();
14495 if (SRL.getOperand(0) !=
SDValue(SMULLOHI, 0) ||
14496 SHL.getOperand(0) !=
SDValue(SMULLOHI, 1))
14515 unsigned Opcode = 0;
14516 if (
isS16(OpS16, DAG))
14534 if (Subtarget->
isThumb1Only() || !Subtarget->hasV6T2Ops())
14537 EVT VT =
N->getValueType(0);
14552 if (VT != MVT::i32)
14565 if (Mask == 0xffff)
14572 if ((Val & ~Mask) != Val)
14597 (Mask == ~Mask2)) {
14600 if (Subtarget->hasDSP() &&
14601 (Mask == 0xffff || Mask == 0xffff0000))
14614 (~Mask == Mask2)) {
14617 if (Subtarget->hasDSP() &&
14618 (Mask2 == 0xffff || Mask2 == 0xffff0000))
14691 EVT VT =
N->getValueType(0);
14696 auto IsFreelyInvertable = [&](
SDValue V) {
14703 if (!(IsFreelyInvertable(N0) || IsFreelyInvertable(N1)))
14719 EVT VT =
N->getValueType(0);
14725 if (Subtarget->hasMVEIntegerOps() && (VT == MVT::v2i1 || VT == MVT::v4i1 ||
14726 VT == MVT::v8i1 || VT == MVT::v16i1))
14729 APInt SplatBits, SplatUndef;
14730 unsigned SplatBitSize;
14732 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14733 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14734 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14735 SplatBitSize == 64) {
14771 unsigned SplatBitSize;
14774 APInt SplatBits0, SplatBits1;
14778 if (BVN0 && BVN0->
isConstantSplat(SplatBits0, SplatUndef, SplatBitSize,
14779 HasAnyUndefs) && !HasAnyUndefs) {
14780 if (BVN1 && BVN1->
isConstantSplat(SplatBits1, SplatUndef, SplatBitSize,
14781 HasAnyUndefs) && !HasAnyUndefs) {
14786 SplatBits0 == ~SplatBits1) {
14816 EVT VT =
N->getValueType(0);
14831 if (Subtarget->hasMVEIntegerOps()) {
14862 ToMask = ~N->getConstantOperandAPInt(2);
14882 unsigned LastActiveBitInA =
A.countr_zero();
14883 unsigned FirstActiveBitInB =
B.getBitWidth() -
B.countl_zero() - 1;
14884 return LastActiveBitInA - 1 == FirstActiveBitInB;
14889 APInt ToMask, FromMask;
14897 APInt NewToMask, NewFromMask;
14899 if (NewFrom !=
From)
14903 if ((NewToMask & ToMask).getBoolValue())
14928 unsigned InvMask =
N->getConstantOperandVal(2);
14932 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
14933 "undefined behavior");
14934 unsigned Mask = (1u << Width) - 1;
14936 if ((Mask & (~Mask2)) == 0)
14945 APInt ToMask1, FromMask1;
14948 APInt ToMask2, FromMask2;
14954 APInt NewFromMask = FromMask1 | FromMask2;
14955 APInt NewToMask = ToMask1 | ToMask2;
14957 EVT VT =
N->getValueType(0);
14960 if (NewFromMask[0] == 0)
14972 APInt ToMask1 = ~N->getConstantOperandAPInt(2);
14973 APInt ToMask2 = ~N0.getConstantOperandAPInt(2);
14975 if (!N0.
hasOneUse() || (ToMask1 & ToMask2) != 0 ||
14979 EVT VT =
N->getValueType(0);
14996 SDValue CSInc = Cmp->getOperand(0);
15046 if (
N->getConstantOperandVal(2) ==
ARMCC::EQ)
15047 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
15050 if (
N->getConstantOperandVal(2) ==
ARMCC::NE)
15052 N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
15080 SDValue BasePtr = LD->getBasePtr();
15082 DAG.
getLoad(MVT::i32,
DL, LD->getChain(), BasePtr, LD->getPointerInfo(),
15083 LD->getAlign(), LD->getMemOperand()->getFlags());
15089 LD->getPointerInfo().getWithOffset(4),
15091 LD->getMemOperand()->getFlags());
15107 bool BVSwap = BV.
getOpcode() == ISD::BITCAST;
15112 BVSwap = BV.
getOpcode() == ISD::BITCAST;
15124 if (!Subtarget->
isLittle() && BVSwap)
15142 if (!Subtarget->
isLittle() && BVSwap)
15185 if (Op0->
getOpcode() == ISD::BITCAST) {
15187 if (Copy.getValueType() == MVT::f32 &&
15189 bool HasGlue = Copy->getNumOperands() == 3;
15190 SDValue Ops[] = {Copy->getOperand(0), Copy->getOperand(1),
15191 HasGlue ? Copy->getOperand(2) :
SDValue()};
15192 EVT OutTys[] = {
N->getValueType(0), MVT::Other, MVT::Glue};
15211 if (LN0->hasOneUse() && LN0->isUnindexed() &&
15212 LN0->getMemoryVT() == MVT::i16) {
15215 LN0->getBasePtr(), LN0->getMemOperand());
15233 EVT VT =
N->getValueType(0);
15267 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
15268 for (
unsigned i = 0; i < NumElts; ++i) {
15286 if (
N->getNumOperands() == 2)
15292 EVT VT =
N->getValueType(0);
15298 for (
unsigned i = 0; i < NumElts; ++i) {
15299 SDValue V = DAG.
getNode(ISD::BITCAST, dl, MVT::f64,
N->getOperand(i));
15306 return DAG.
getNode(ISD::BITCAST, dl, VT, BV);
15324 EVT VT =
N->getValueType(0);
15332 assert(EltVT == MVT::f32 &&
"Unexpected type!");
15336 if (
Use->getOpcode() != ISD::BITCAST ||
15337 Use->getValueType(0).isFloatingPoint())
15345 unsigned NumOfBitCastedElts = 0;
15347 unsigned NumOfRelevantElts = NumElts;
15348 for (
unsigned Idx = 0; Idx < NumElts; ++Idx) {
15350 if (Elt->
getOpcode() == ISD::BITCAST) {
15353 ++NumOfBitCastedElts;
15357 --NumOfRelevantElts;
15361 if (NumOfBitCastedElts <= NumOfRelevantElts / 2)
15379 for (
unsigned Idx = 0 ; Idx < NumElts; ++Idx) {
15383 if (V.getOpcode() == ISD::BITCAST &&
15384 V->getOperand(0).getValueType() == MVT::i32)
15395 Vec = DAG.
getNode(ISD::BITCAST, dl, VT, Vec);
15403 EVT VT =
N->getValueType(0);
15410 if (
Op->getOperand(0).getValueType() == VT)
15411 return Op->getOperand(0);
15426 if (
Op.getValueType() == MVT::i32) {
15437 EVT VT =
N->getValueType(0);
15442 if (ST->isLittle())
15443 return DAG.
getNode(ISD::BITCAST, dl, VT,
Op);
15446 if (
Op.getValueType() == VT)
15455 if (
Op->getOperand(0).getValueType() == VT)
15456 return Op->getOperand(0);
15465 if (!Subtarget->hasMVEIntegerOps())
15468 EVT VT =
N->getValueType(0);
15499 EVT VT =
N->getValueType(0);
15509 SDValue Vec = DAG.
getNode(ISD::BITCAST, dl, FloatVT,
N->getOperand(0));
15510 SDValue V = DAG.
getNode(ISD::BITCAST, dl, MVT::f64,
N->getOperand(1));
15516 return DAG.
getNode(ISD::BITCAST, dl, VT, InsElt);
15525 EVT VT =
N->getValueType(0);
15533 if (Ext.getOpcode() == ISD::BITCAST &&
15534 Ext.getOperand(0).getValueType() == MVT::f32)
15535 Ext = Ext.getOperand(0);
15538 Ext.getConstantOperandVal(1) % 2 != 0)
15540 if (Ext->use_size() == 1 &&
15545 SDValue Op0 = Ext.getOperand(0);
15548 unsigned Lane = Ext.getConstantOperandVal(1);
15554 return V->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15555 isa<ConstantSDNode>(V->getOperand(1)) &&
15556 V->getConstantOperandVal(1) == Lane + 1 &&
15557 V->getOperand(0).getResNo() == ResNo;
15559 if (OtherIt == Op0->
uses().end())
15564 SDValue OtherExt(*OtherIt, 0);
15577 DCI.
DAG.
getConstant(Ext.getConstantOperandVal(1) / 2, dl, MVT::i32));
15589 EVT VT =
N->getValueType(0);
15595 if (VT == MVT::f16 &&
X.getValueType() == MVT::i32)
15597 if (VT == MVT::i32 &&
X.getValueType() == MVT::f16)
15599 if (VT == MVT::f32 &&
X.getValueType() == MVT::i32)
15602 while (
X.getValueType() != VT &&
X->getOpcode() == ISD::BITCAST)
15604 if (
X.getValueType() == VT)
15612 return Op0.
getOperand(
N->getConstantOperandVal(1));
15634 unsigned Idx =
N->getConstantOperandVal(1);
15648 EVT VT =
N->getValueType(0);
15653 Op.getOperand(0).getValueType().getScalarType())
15664 uint64_t IdxVal =
N->getConstantOperandVal(2);
15675 if (IdxVal == 0 && Vec.
isUndef())
15681 (IdxVal != 0 && IdxVal != NumSubElts))
15754 EVT VT =
N->getValueType(0);
15765 unsigned HalfElts = NumElts/2;
15767 for (
unsigned n = 0; n < NumElts; ++n) {
15770 if (MaskElt < (
int)HalfElts)
15772 else if (MaskElt >= (
int)NumElts && MaskElt < (
int)(NumElts + HalfElts))
15773 NewElt = HalfElts + MaskElt - NumElts;
15800 bool SimpleConstIncOnly,
15808 bool isLoadOp =
true;
15809 bool isLaneOp =
false;
15812 bool hasAlignment =
true;
15813 unsigned NewOpc = 0;
15814 unsigned NumVecs = 0;
15815 if (
Target.isIntrinsic) {
15816 unsigned IntNo =
N->getConstantOperandVal(1);
15820 case Intrinsic::arm_neon_vld1:
15824 case Intrinsic::arm_neon_vld2:
15828 case Intrinsic::arm_neon_vld3:
15832 case Intrinsic::arm_neon_vld4:
15836 case Intrinsic::arm_neon_vld1x2:
15839 hasAlignment =
false;
15841 case Intrinsic::arm_neon_vld1x3:
15844 hasAlignment =
false;
15846 case Intrinsic::arm_neon_vld1x4:
15849 hasAlignment =
false;
15851 case Intrinsic::arm_neon_vld2dup:
15855 case Intrinsic::arm_neon_vld3dup:
15859 case Intrinsic::arm_neon_vld4dup:
15863 case Intrinsic::arm_neon_vld2lane:
15868 case Intrinsic::arm_neon_vld3lane:
15873 case Intrinsic::arm_neon_vld4lane:
15878 case Intrinsic::arm_neon_vst1:
15883 case Intrinsic::arm_neon_vst2:
15888 case Intrinsic::arm_neon_vst3:
15893 case Intrinsic::arm_neon_vst4:
15898 case Intrinsic::arm_neon_vst2lane:
15904 case Intrinsic::arm_neon_vst3lane:
15910 case Intrinsic::arm_neon_vst4lane:
15916 case Intrinsic::arm_neon_vst1x2:
15920 hasAlignment =
false;
15922 case Intrinsic::arm_neon_vst1x3:
15926 hasAlignment =
false;
15928 case Intrinsic::arm_neon_vst1x4:
15932 hasAlignment =
false;
15937 switch (
N->getOpcode()) {
15973 VecTy =
N->getValueType(0);
15974 }
else if (
Target.isIntrinsic) {
15975 VecTy =
N->getOperand(
Target.AddrOpIdx + 1).getValueType();
15978 "Node has to be a load, a store, or an intrinsic!");
15979 VecTy =
N->getOperand(1).getValueType();
15987 if (isLaneOp || isVLDDUPOp)
15990 if (NumBytes >= 3 * 16 &&
User.ConstInc != NumBytes) {
15996 if (SimpleConstIncOnly &&
User.ConstInc != NumBytes)
16002 EVT AlignedVecTy = VecTy;
16022 assert(NumVecs == 1 &&
"Unexpected multi-element generic load/store.");
16023 assert(!isLaneOp &&
"Unexpected generic load/store lane.");
16034 Alignment =
Align(1);
16040 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16042 for (n = 0; n < NumResultVecs; ++n)
16043 Tys[n] = AlignedVecTy;
16044 Tys[n++] = MVT::i32;
16045 Tys[n] = MVT::Other;
16060 unsigned LastOperand =
16061 hasAlignment ?
N->getNumOperands() - 1 :
N->getNumOperands();
16062 for (
unsigned i =
Target.AddrOpIdx + 1; i < LastOperand; ++i)
16071 if (AlignedVecTy != VecTy &&
N->getOpcode() == ISD::STORE) {
16073 StVal = DAG.
getNode(ISD::BITCAST, dl, AlignedVecTy, StVal);
16082 for (
unsigned i = 0; i < NumResultVecs; ++i)
16087 if (AlignedVecTy != VecTy &&
N->getOpcode() == ISD::LOAD) {
16088 SDValue &LdVal = NewResults[0];
16089 LdVal = DAG.
getNode(ISD::BITCAST, dl, VecTy, LdVal);
16124 switch (
N->getOpcode()) {
16128 *
Ptr =
N->getOperand(0);
16136 *
Ptr =
N->getOperand(1);
16171 const bool isStore =
N->getOpcode() == ISD::STORE;
16172 const unsigned AddrOpIdx = ((isIntrinsic ||
isStore) ? 2 : 1);
16181 UE =
Addr.getNode()->use_end(); UI != UE; ++UI) {
16183 if (UI.getUse().getResNo() !=
Addr.getResNo() ||
16188 unsigned ConstInc =
16206 if (UI.getUse().getResNo() !=
Base.getResNo() ||
User ==
Addr.getNode() ||
16211 unsigned UserOffset =
16214 if (!UserOffset || UserOffset <=
Offset)
16217 unsigned NewConstInc = UserOffset -
Offset;
16227 unsigned NumValidUpd = BaseUpdates.
size();
16228 for (
unsigned I = 0;
I < NumValidUpd;) {
16232 std::swap(BaseUpdates[
I], BaseUpdates[NumValidUpd]);
16240 BaseUpdates.
resize(NumValidUpd);
16245 std::stable_sort(BaseUpdates.
begin(), BaseUpdates.
end(),
16247 return LHS.ConstInc < RHS.ConstInc;
16276 unsigned IntNo =
N->getConstantOperandVal(1);
16277 if (IntNo == Intrinsic::arm_mve_vst2q &&
N->getConstantOperandVal(5) != 1)
16279 if (IntNo == Intrinsic::arm_mve_vst4q &&
N->getConstantOperandVal(7) != 3)
16284 UE =
Addr.getNode()->use_end();
16288 UI.getUse().getResNo() !=
Addr.getResNo())
16304 bool isLoadOp =
true;
16305 unsigned NewOpc = 0;
16306 unsigned NumVecs = 0;
16310 case Intrinsic::arm_mve_vld2q:
16314 case Intrinsic::arm_mve_vld4q:
16318 case Intrinsic::arm_mve_vst2q:
16323 case Intrinsic::arm_mve_vst4q:
16333 VecTy =
N->getValueType(0);
16335 VecTy =
N->getOperand(3).getValueType();
16349 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16351 for (n = 0; n < NumResultVecs; ++n)
16353 Tys[n++] = MVT::i32;
16354 Tys[n] = MVT::Other;
16363 for (
unsigned i = 3; i <
N->getNumOperands(); ++i)
16371 for (
unsigned i = 0; i < NumResultVecs; ++i)
16390 EVT VT =
N->getValueType(0);
16399 unsigned NumVecs = 0;
16400 unsigned NewOpc = 0;
16402 if (IntNo == Intrinsic::arm_neon_vld2lane) {
16405 }
else if (IntNo == Intrinsic::arm_neon_vld3lane) {
16408 }
else if (IntNo == Intrinsic::arm_neon_vld4lane) {
16421 if (UI.getUse().getResNo() == NumVecs)
16425 VLDLaneNo !=
User->getConstantOperandVal(1))
16432 for (n = 0; n < NumVecs; ++n)
16434 Tys[n] = MVT::Other;
16445 unsigned ResNo = UI.getUse().getResNo();
16447 if (ResNo == NumVecs)
16455 std::vector<SDValue> VLDDupResults;
16456 for (
unsigned n = 0; n < NumVecs; ++n)
16470 EVT VT =
N->getValueType(0);
16473 if (Subtarget->hasMVEIntegerOps()) {
16477 ExtractVT = MVT::i32;
16479 N->getOperand(0),
N->getOperand(1));
16490 while (
Op.getOpcode() == ISD::BITCAST)
16491 Op =
Op.getOperand(0);
16496 unsigned EltSize =
Op.getScalarValueSizeInBits();
16498 unsigned Imm =
Op.getConstantOperandVal(0);
16514 if (Subtarget->hasMVEIntegerOps()) {
16517 if (
Op.getValueType() == MVT::f32)
16519 DAG.
getNode(ISD::BITCAST, dl, MVT::i32,
Op));
16520 else if (
Op.getValueType() == MVT::f16)
16525 if (!Subtarget->hasNEON())
16532 if (LD &&
Op.hasOneUse() && LD->isUnindexed() &&
16533 LD->getMemoryVT() ==
N->getValueType(0).getVectorElementType()) {
16534 SDValue Ops[] = {LD->getOperand(0), LD->getOperand(1),
16539 LD->getMemoryVT(), LD->getMemOperand());
16550 EVT VT =
N->getValueType(0);
16572 assert(StVT != VT &&
"Cannot truncate to the same type");
16582 if (0 != (NumElems * FromEltSz) % ToEltSz)
16585 unsigned SizeRatio = FromEltSz / ToEltSz;
16590 NumElems * SizeRatio);
16596 for (
unsigned i = 0; i < NumElems; ++i)
16610 MVT StoreType = MVT::i8;
16612 if (TLI.
isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz)
16631 unsigned E = (ToEltSz * NumElems) / StoreType.
getSizeInBits();
16632 for (
unsigned I = 0;
I <
E;
I++) {
16663 if (FromEltVT != MVT::f32 || ToEltVT != MVT::f16)
16666 unsigned NumElements = 4;
16683 unsigned Off0 = Rev ? NumElts : 0;
16684 unsigned Off1 = Rev ? 0 : NumElts;
16686 for (
unsigned I = 0;
I < NumElts;
I += 2) {
16687 if (M[
I] >= 0 && M[
I] != (
int)(Off0 +
I / 2))
16689 if (M[
I + 1] >= 0 && M[
I + 1] != (
int)(Off1 +
I / 2))
16697 if (isVMOVNShuffle(Shuffle,
false) || isVMOVNShuffle(Shuffle,
true))
16717 unsigned NewOffset = i * NumElements * ToEltVT.
getSizeInBits() / 8;
16732 NewToVT, Alignment, MMOFlags, AAInfo);
16765 unsigned NewOffset =
16773 NewToVT, Alignment, MMOFlags, AAInfo);
16795 {Extract.getOperand(0), Extract.getOperand(1)});
16826 if (Subtarget->hasNEON())
16830 if (Subtarget->hasMVEFloatOps())
16834 if (Subtarget->hasMVEIntegerOps()) {
16910 if (!Subtarget->hasNEON())
16914 if (!
Op.getValueType().isVector() || !
Op.getValueType().isSimple() ||
16922 MVT FloatTy =
Op.getSimpleValueType().getVectorElementType();
16925 uint32_t IntBits = IntTy.getSizeInBits();
16926 unsigned NumLanes =
Op.getValueType().getVectorNumElements();
16927 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
16938 if (
C == -1 ||
C == 0 ||
C > 32)
16943 unsigned IntrinsicOpcode =
isSigned ? Intrinsic::arm_neon_vcvtfp2fxs :
16944 Intrinsic::arm_neon_vcvtfp2fxu;
16947 DAG.
getConstant(IntrinsicOpcode, dl, MVT::i32),
Op->getOperand(0),
16950 if (IntBits < FloatBits)
16958 if (!Subtarget->hasMVEFloatOps())
16966 EVT VT =
N->getValueType(0);
16971 auto isIdentitySplat = [&](
SDValue Op,
bool NSZ) {
16972 if (
Op.getOpcode() != ISD::BITCAST ||
16975 uint64_t ImmVal =
Op.getOperand(0).getConstantOperandVal(0);
16976 if (VT == MVT::v4f32 && (ImmVal == 1664 || (ImmVal == 0 && NSZ)))
16978 if (VT == MVT::v8f16 && (ImmVal == 2688 || (ImmVal == 0 && NSZ)))
16991 if (!isIdentitySplat(Op1.
getOperand(2), NSZ))
17002 EVT VT =
N->getValueType(0);
17005 if (!
N->getFlags().hasAllowReassociation())
17012 unsigned Opc =
A.getConstantOperandVal(0);
17013 if (Opc != Intrinsic::arm_mve_vcmlaq)
17022 if (
SDValue R = ReassocComplex(LHS, RHS))
17024 if (
SDValue R = ReassocComplex(RHS, LHS))
17050 if (!Subtarget->hasNEON())
17054 unsigned OpOpcode =
Op.getNode()->getOpcode();
17055 if (!
N->getValueType(0).isVector() || !
N->getValueType(0).isSimple() ||
17065 MVT IntTy =
Op.getOperand(0).getSimpleValueType().getVectorElementType();
17066 uint32_t IntBits = IntTy.getSizeInBits();
17067 unsigned NumLanes =
Op.getValueType().getVectorNumElements();
17068 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
17088 int32_t
C = IntVal.exactLogBase2();
17089 if (
C == -1 ||
C == 0 ||
C > 32)
17095 if (IntBits < FloatBits)
17097 NumLanes == 2 ? MVT::v2i32 : MVT::v4i32, ConvInput);
17099 unsigned IntrinsicOpcode =
isSigned ? Intrinsic::arm_neon_vcvtfxs2fp
17100 : Intrinsic::arm_neon_vcvtfxu2fp;
17108 if (!ST->hasMVEIntegerOps())
17111 assert(
N->getOpcode() == ISD::VECREDUCE_ADD);
17112 EVT ResVT =
N->getValueType(0);
17140 EVT AVT =
A.getValueType();
17146 auto ExtendIfNeeded = [&](
SDValue A,
unsigned ExtendCode) {
17147 EVT AVT =
A.getValueType();
17155 auto IsVADDV = [&](
MVT RetTy,
unsigned ExtendCode,
ArrayRef<MVT> ExtTypes) {
17156 if (ResVT != RetTy || N0->
getOpcode() != ExtendCode)
17159 if (ExtTypeMatches(
A, ExtTypes))
17160 return ExtendIfNeeded(
A, ExtendCode);
17163 auto IsPredVADDV = [&](
MVT RetTy,
unsigned ExtendCode,
17170 if (Ext->getOpcode() != ExtendCode)
17173 if (ExtTypeMatches(
A, ExtTypes))
17174 return ExtendIfNeeded(
A, ExtendCode);
17177 auto IsVMLAV = [&](
MVT RetTy,
unsigned ExtendCode,
ArrayRef<MVT> ExtTypes,
17187 if (ResVT != RetTy)
17190 if (
Mul->getOpcode() == ExtendCode &&
17191 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17202 if (ExtTypeMatches(
A, ExtTypes) && ExtTypeMatches(
B, ExtTypes)) {
17203 A = ExtendIfNeeded(
A, ExtendCode);
17204 B = ExtendIfNeeded(
B, ExtendCode);
17209 auto IsPredVMLAV = [&](
MVT RetTy,
unsigned ExtendCode,
ArrayRef<MVT> ExtTypes,
17222 if (
Mul->getOpcode() == ExtendCode &&
17223 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17234 if (ExtTypeMatches(
A, ExtTypes) && ExtTypeMatches(
B, ExtTypes)) {
17235 A = ExtendIfNeeded(
A, ExtendCode);
17236 B = ExtendIfNeeded(
B, ExtendCode);
17247 EVT VT = Ops[0].getValueType();
17248 if (VT == MVT::v16i8) {
17250 "Unexpected illegal long reduction opcode");
17255 DAG.
getVTList(MVT::v8i16, MVT::v8i16), Ops[0]);
17258 DAG.
getVTList(MVT::v8i16, MVT::v8i16), Ops[1]);
17279 if (IsVMLAV(MVT::i64,
ISD::SIGN_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17282 if (IsVMLAV(MVT::i64,
ISD::ZERO_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17346 Op =
Op->getOperand(1);
17348 Op->getOperand(0)->getOpcode() ==
ISD::MUL) {
17350 if (
Mul->getOperand(0) ==
Mul->getOperand(1) &&
17356 return DAG.
getNode(ISD::VECREDUCE_ADD, dl, ResVT, Ext);
17369 if (!Shuf || !Shuf->getOperand(1).isUndef())
17374 APInt SetElts(Mask.size(), 0);
17375 for (
int E : Mask) {
17383 if (
N->getNumOperands() != VecOp + 1) {
17385 if (!Shuf2 || !Shuf2->getOperand(1).isUndef() || Shuf2->getMask() != Mask)
17391 if (
Op.getValueType().isVector())
17410 if (Op0->
isUndef() && !IsTop)
17426 APInt Op0DemandedElts =
17427 IsTop ? Op1DemandedElts
17444 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
17445 APInt Op0DemandedElts =
17457 EVT VT =
N->getValueType(0);
17464 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(Shuf1->getMask()) &&
17465 LHS.getOperand(1).isUndef() && RHS.getOperand(1).isUndef() &&
17466 (LHS.hasOneUse() || RHS.hasOneUse() || LHS == RHS)) {
17469 LHS.getOperand(0), RHS.getOperand(0));
17484 int ShiftAmt =
C->getSExtValue();
17485 if (ShiftAmt == 0) {
17491 if (ShiftAmt >= -32 && ShiftAmt < 0) {
17492 unsigned NewOpcode =
17508 unsigned IntNo =
N->getConstantOperandVal(0);
17519 case Intrinsic::arm_neon_vshifts:
17520 case Intrinsic::arm_neon_vshiftu:
17521 case Intrinsic::arm_neon_vrshifts:
17522 case Intrinsic::arm_neon_vrshiftu:
17523 case Intrinsic::arm_neon_vrshiftn:
17524 case Intrinsic::arm_neon_vqshifts:
17525 case Intrinsic::arm_neon_vqshiftu:
17526 case Intrinsic::arm_neon_vqshiftsu:
17527 case Intrinsic::arm_neon_vqshiftns:
17528 case Intrinsic::arm_neon_vqshiftnu:
17529 case Intrinsic::arm_neon_vqshiftnsu:
17530 case Intrinsic::arm_neon_vqrshiftns:
17531 case Intrinsic::arm_neon_vqrshiftnu:
17532 case Intrinsic::arm_neon_vqrshiftnsu: {
17533 EVT VT =
N->getOperand(1).getValueType();
17535 unsigned VShiftOpc = 0;
17538 case Intrinsic::arm_neon_vshifts:
17539 case Intrinsic::arm_neon_vshiftu:
17544 if (
isVShiftRImm(
N->getOperand(2), VT,
false,
true, Cnt)) {
17551 case Intrinsic::arm_neon_vrshifts:
17552 case Intrinsic::arm_neon_vrshiftu:
17557 case Intrinsic::arm_neon_vqshifts:
17558 case Intrinsic::arm_neon_vqshiftu:
17563 case Intrinsic::arm_neon_vqshiftsu:
17568 case Intrinsic::arm_neon_vrshiftn:
17569 case Intrinsic::arm_neon_vqshiftns:
17570 case Intrinsic::arm_neon_vqshiftnu:
17571 case Intrinsic::arm_neon_vqshiftnsu:
17572 case Intrinsic::arm_neon_vqrshiftns:
17573 case Intrinsic::arm_neon_vqrshiftnu:
17574 case Intrinsic::arm_neon_vqrshiftnsu:
17586 case Intrinsic::arm_neon_vshifts:
17587 case Intrinsic::arm_neon_vshiftu:
17590 case Intrinsic::arm_neon_vrshifts:
17593 case Intrinsic::arm_neon_vrshiftu:
17596 case Intrinsic::arm_neon_vrshiftn:
17599 case Intrinsic::arm_neon_vqshifts:
17602 case Intrinsic::arm_neon_vqshiftu:
17605 case Intrinsic::arm_neon_vqshiftsu:
17608 case Intrinsic::arm_neon_vqshiftns:
17611 case Intrinsic::arm_neon_vqshiftnu:
17614 case Intrinsic::arm_neon_vqshiftnsu:
17617 case Intrinsic::arm_neon_vqrshiftns:
17620 case Intrinsic::arm_neon_vqrshiftnu:
17623 case Intrinsic::arm_neon_vqrshiftnsu:
17629 return DAG.
getNode(VShiftOpc, dl,
N->getValueType(0),
17630 N->getOperand(1), DAG.
getConstant(Cnt, dl, MVT::i32));
17633 case Intrinsic::arm_neon_vshiftins: {
17634 EVT VT =
N->getOperand(1).getValueType();
17636 unsigned VShiftOpc = 0;
17640 else if (
isVShiftRImm(
N->getOperand(3), VT,
false,
true, Cnt))
17647 return DAG.
getNode(VShiftOpc, dl,
N->getValueType(0),
17648 N->getOperand(1),
N->getOperand(2),
17652 case Intrinsic::arm_neon_vqrshifts:
17653 case Intrinsic::arm_neon_vqrshiftu:
17657 case Intrinsic::arm_neon_vbsl: {
17660 N->getOperand(2),
N->getOperand(3));
17662 case Intrinsic::arm_mve_vqdmlah:
17663 case Intrinsic::arm_mve_vqdmlash:
17664 case Intrinsic::arm_mve_vqrdmlah:
17665 case Intrinsic::arm_mve_vqrdmlash:
17666 case Intrinsic::arm_mve_vmla_n_predicated:
17667 case Intrinsic::arm_mve_vmlas_n_predicated:
17668 case Intrinsic::arm_mve_vqdmlah_predicated:
17669 case Intrinsic::arm_mve_vqdmlash_predicated:
17670 case Intrinsic::arm_mve_vqrdmlah_predicated:
17671 case Intrinsic::arm_mve_vqrdmlash_predicated: {
17676 unsigned BitWidth =
N->getValueType(0).getScalarSizeInBits();
17683 case Intrinsic::arm_mve_minv:
17684 case Intrinsic::arm_mve_maxv:
17685 case Intrinsic::arm_mve_minav:
17686 case Intrinsic::arm_mve_maxav:
17687 case Intrinsic::arm_mve_minv_predicated:
17688 case Intrinsic::arm_mve_maxv_predicated:
17689 case Intrinsic::arm_mve_minav_predicated:
17690 case Intrinsic::arm_mve_maxav_predicated: {
17693 unsigned BitWidth =
N->getOperand(2)->getValueType(0).getScalarSizeInBits();
17700 case Intrinsic::arm_mve_addv: {
17703 bool Unsigned =
N->getConstantOperandVal(2);
17708 case Intrinsic::arm_mve_addlv:
17709 case Intrinsic::arm_mve_addlv_predicated: {
17712 bool Unsigned =
N->getConstantOperandVal(2);
17713 unsigned Opc = IntNo == Intrinsic::arm_mve_addlv ?
17718 for (
unsigned i = 1, e =
N->getNumOperands(); i < e; i++)
17741 EVT VT =
N->getValueType(0);
17743 if (ST->isThumb1Only() &&
N->getOpcode() ==
ISD::SHL && VT == MVT::i32 &&
17744 N->getOperand(0)->getOpcode() ==
ISD::AND &&
17745 N->getOperand(0)->hasOneUse()) {
17762 if (AndMask == 255 || AndMask == 65535)
17766 if (MaskedBits > ShiftAmt) {
17781 if (ST->hasMVEIntegerOps())
17786 switch (
N->getOpcode()) {
17799 if (
isVShiftRImm(
N->getOperand(1), VT,
false,
false, Cnt)) {
17800 unsigned VShiftOpc =
17803 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0),
17819 if (!LD->isSimple() || !N0.
hasOneUse() || LD->isIndexed() ||
17822 EVT FromVT = LD->getValueType(0);
17823 EVT ToVT =
N->getValueType(0);
17830 unsigned NumElements = 0;
17831 if (ToEltVT == MVT::i32 && FromEltVT == MVT::i8)
17833 if (ToEltVT == MVT::f32 && FromEltVT == MVT::f16)
17835 if (NumElements == 0 ||
17845 SDValue BasePtr = LD->getBasePtr();
17846 Align Alignment = LD->getOriginalAlign();
17867 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
17868 Alignment, MMOFlags, AAInfo);
17874 if (FromEltVT == MVT::f16) {
17877 for (
unsigned i = 0; i < Loads.
size(); i++) {
17903 if ((ST->hasNEON() || ST->hasMVEIntegerOps()) &&
17907 EVT VT =
N->getValueType(0);
17911 if (VT == MVT::i32 &&
17912 (EltVT == MVT::i8 || EltVT == MVT::i16) &&
17917 switch (
N->getOpcode()) {
17931 if (ST->hasMVEIntegerOps())
17940 if (ST->hasMVEFloatOps())
17951 if ((Subtarget->isThumb() || !Subtarget->hasV6Ops()) &&
17955 EVT VT =
Op.getValueType();
17958 if (VT != MVT::i32 ||
17971 APInt MaxC = Max.getConstantOperandAPInt(1);
17974 !(MinC + 1).isPowerOf2())
17992 EVT VT =
N->getValueType(0);
17995 if (VT == MVT::i32)
17998 if (!ST->hasMVEIntegerOps())
18004 if (VT != MVT::v4i32 && VT != MVT::v8i16)
18007 auto IsSignedSaturate = [&](
SDNode *Min,
SDNode *Max) {
18015 if (VT == MVT::v4i32)
18016 SaturateC =
APInt(32, (1 << 15) - 1,
true);
18018 SaturateC =
APInt(16, (1 << 7) - 1,
true);
18025 MaxC != ~SaturateC)
18030 if (IsSignedSaturate(
N, N0.
getNode())) {
18033 if (VT == MVT::v4i32) {
18034 HalfVT = MVT::v8i16;
18035 ExtVT = MVT::v4i16;
18037 HalfVT = MVT::v16i8;
18052 auto IsUnsignedSaturate = [&](
SDNode *Min) {
18058 if (VT == MVT::v4i32)
18059 SaturateC =
APInt(32, (1 << 16) - 1,
true);
18061 SaturateC =
APInt(16, (1 << 8) - 1,
true);
18070 if (IsUnsignedSaturate(
N)) {
18074 if (VT == MVT::v4i32) {
18075 HalfVT = MVT::v8i16;
18076 ExtConst = 0x0000FFFF;
18078 HalfVT = MVT::v16i8;
18100 const APInt *CV = &
C->getAPIntValue();
18118 SDValue Op0 = CMOV->getOperand(0);
18119 SDValue Op1 = CMOV->getOperand(1);
18120 auto CC = CMOV->getConstantOperandAPInt(2).getLimitedValue();
18121 SDValue CmpZ = CMOV->getOperand(4);
18157 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2;
18164 if ((OrCI & Known.
Zero) != OrCI)
18170 EVT VT =
X.getValueType();
18171 unsigned BitInX = AndC->
logBase2();
18179 for (
unsigned BitInY = 0, NumActiveBits = OrCI.
getActiveBits();
18180 BitInY < NumActiveBits; ++BitInY) {
18181 if (OrCI[BitInY] == 0)
18184 Mask.setBit(BitInY);
18201 switch (
N->getOpcode()) {
18216 if (Const->isZero())
18218 else if (Const->isOne())
18226 unsigned IntOp =
N.getConstantOperandVal(1);
18227 if (IntOp != Intrinsic::test_start_loop_iterations &&
18228 IntOp != Intrinsic::loop_decrement_reg)
18254 bool Negate =
false;
18258 if (
N->getOpcode() == ISD::BRCOND) {
18260 Cond =
N->getOperand(1);
18263 assert(
N->getOpcode() == ISD::BR_CC &&
"Expected BRCOND or BR_CC!");
18265 Cond =
N->getOperand(2);
18268 if (!Const->isOne() && !Const->isZero())
18270 Imm = Const->getZExtValue();
18298 assert((IsTrueIfZero(
CC, Imm) || IsFalseIfZero(
CC, Imm)) &&
18299 "unsupported condition");
18304 unsigned IntOp =
Int->getConstantOperandVal(1);
18305 assert((
N->hasOneUse() &&
N->use_begin()->getOpcode() == ISD::BR)
18306 &&
"expected single br user");
18317 if (IntOp == Intrinsic::test_start_loop_iterations) {
18321 if (IsTrueIfZero(
CC, Imm)) {
18322 SDValue Ops[] = {Chain, Setup, Dest};
18327 UpdateUncondBr(Br, Dest, DAG);
18329 SDValue Ops[] = {Chain, Setup, OtherTarget};
18342 DAG.
getVTList(MVT::i32, MVT::Other), Args);
18350 if (
Target == OtherTarget)
18351 UpdateUncondBr(Br, Dest, DAG);
18370 EVT VT =
N->getValueType(0);
18372 SDValue LHS = Cmp.getOperand(0);
18373 SDValue RHS = Cmp.getOperand(1);
18383 LHS->getOperand(0)->hasOneUse() &&
18403 EVT VT =
N->getValueType(0);
18405 SDValue LHS = Cmp.getOperand(0);
18406 SDValue RHS = Cmp.getOperand(1);
18407 SDValue FalseVal =
N->getOperand(0);
18408 SDValue TrueVal =
N->getOperand(1);
18413 if (!Subtarget->
isThumb1Only() && Subtarget->hasV6T2Ops()) {
18437 if (
CC ==
ARMCC::NE && FalseVal == RHS && FalseVal != LHS) {
18453 LHS->getOperand(2), LHS->getOperand(3),
18454 LHS->getOperand(4));
18464 if (
N->getConstantOperandVal(2) ==
ARMCC::EQ ||
18468 if (
N->getConstantOperandVal(2) ==
ARMCC::NE)
18470 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
18480 if (!Subtarget->
isThumb1Only() && Subtarget->hasV5TOps()) {
18549 const APInt *TrueConst;
18551 ((FalseVal.getOpcode() ==
ARMISD::SUBC && FalseVal.getOperand(0) == LHS &&
18552 FalseVal.getOperand(1) == RHS) ||
18556 unsigned ShiftAmount = TrueConst->
logBase2();
18571 if (Known.
Zero == 0xfffffffe)
18574 else if (Known.
Zero == 0xffffff00)
18577 else if (Known.
Zero == 0xffff0000)
18590 EVT DstVT =
N->getValueType(0);
18593 if (ST->hasMVEIntegerOps() && Src.getOpcode() ==
ARMISD::VDUP) {
18594 EVT SrcVT = Src.getValueType();
18602 Src.getOperand(0).getValueType().getScalarSizeInBits() <=
18603 Src.getValueType().getScalarSizeInBits())
18604 Src = Src.getOperand(0);
18608 EVT SrcVT = Src.getValueType();
18628 EVT VT =
N->getValueType(0);
18636 if (
N->getNumOperands() == 2 &&
18640 N->getOperand(0).getOperand(1),
18641 N->getOperand(1).getOperand(0),
18642 N->getOperand(1).getOperand(1));
18645 if (
N->getNumOperands() == 2 &&
18651 if (S0->getOperand(0) ==
S1->getOperand(0) &&
18652 S0->getOperand(1) ==
S1->getOperand(1)) {
18655 Mask.append(
S1->getMask().begin(),
S1->getMask().end());
18675 return Op.getOpcode() == ISD::BUILD_VECTOR ||
18676 Op.getOpcode() == ISD::VECTOR_SHUFFLE ||
18677 (Op.getOpcode() == ISD::BITCAST &&
18678 Op.getOperand(0).getOpcode() == ISD::BUILD_VECTOR);
18683 for (
unsigned i = 0; i < O.getValueType().getVectorNumElements(); i++) {
18702 assert((NumIns == 2 || NumIns == 4) &&
18703 "Expected 2 or 4 inputs to an MVETrunc");
18705 if (
N->getNumOperands() == 4)
18709 for (
int I = 0;
I < NumIns;
I++) {
18711 ISD::ADD,
DL, StackPtr.getValueType(), StackPtr,
18731 if (!LD || !LD->isSimple() || !N0.
hasOneUse() || LD->isIndexed())
18734 EVT FromVT = LD->getMemoryVT();
18735 EVT ToVT =
N->getValueType(0);
18742 unsigned NumElements = 0;
18743 if (ToEltVT == MVT::i32 && (FromEltVT == MVT::i16 || FromEltVT == MVT::i8))
18745 if (ToEltVT == MVT::i16 && FromEltVT == MVT::i8)
18747 assert(NumElements != 0);
18753 LD->getExtensionType() != NewExtType)
18760 SDValue BasePtr = LD->getBasePtr();
18761 Align Alignment = LD->getOriginalAlign();
18780 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
18781 Alignment, MMOFlags, AAInfo);
18797 EVT VT =
N->getValueType(0);
18799 assert(
N->getNumValues() == 2 &&
"Expected MVEEXT with 2 elements");
18800 assert((VT == MVT::v4i32 || VT == MVT::v8i16) &&
"Unexpected MVEEXT type");
18804 auto Extend = [&](
SDValue V) {
18814 SDValue Ext = Extend(
N->getOperand(0));
18822 assert(Mask.size() == SVN->getValueType(0).getVectorNumElements());
18827 auto CheckInregMask = [&](
int Start,
int Offset) {
18829 if (Mask[Start + Idx] >= 0 && Mask[Start + Idx] != Idx * 2 +
Offset)
18835 if (CheckInregMask(0, 0))
18837 else if (CheckInregMask(0, 1))
18838 V0 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op0));
18839 else if (CheckInregMask(0, Mask.size()))
18841 else if (CheckInregMask(0, Mask.size() + 1))
18842 V0 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op1));
18847 V1 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op1));
18851 V1 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op0));
18858 if (
N->getOperand(0)->getOpcode() == ISD::LOAD)
18870 assert((NumOuts == 2 || NumOuts == 4) &&
18871 "Expected 2 or 4 outputs to an MVEEXT");
18874 if (
N->getNumOperands() == 4)
18880 StackPtr, MPI,
Align(4));
18883 for (
int I = 0;
I < NumOuts;
I++) {
18885 ISD::ADD,
DL, StackPtr.getValueType(), StackPtr,
18886 DAG.
getConstant(
I * 16 / NumOuts,
DL, StackPtr.getValueType()));
18891 VT, Chain,
Ptr, MPI, LoadVT,
Align(4));
18900 switch (
N->getOpcode()) {
18951 case ISD::FP_EXTEND:
18990 case ISD::VECREDUCE_ADD:
19017 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19024 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19035 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19043 unsigned LowWidth =
N->getOperand(0).getValueType().getSizeInBits();
19045 unsigned HighWidth =
N->getOperand(1).getValueType().getSizeInBits();
19053 unsigned HighWidth =
N->getOperand(0).getValueType().getSizeInBits();
19055 unsigned LowWidth =
N->getOperand(1).getValueType().getSizeInBits();
19063 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19074 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19082 if (
N->getOperand(1) ==
N->getOperand(2))
19083 return N->getOperand(1);
19087 switch (
N->getConstantOperandVal(1)) {
19088 case Intrinsic::arm_neon_vld1:
19089 case Intrinsic::arm_neon_vld1x2:
19090 case Intrinsic::arm_neon_vld1x3:
19091 case Intrinsic::arm_neon_vld1x4:
19092 case Intrinsic::arm_neon_vld2:
19093 case Intrinsic::arm_neon_vld3:
19094 case Intrinsic::arm_neon_vld4:
19095 case Intrinsic::arm_neon_vld2lane:
19096 case Intrinsic::arm_neon_vld3lane:
19097 case Intrinsic::arm_neon_vld4lane:
19098 case Intrinsic::arm_neon_vld2dup:
19099 case Intrinsic::arm_neon_vld3dup:
19100 case Intrinsic::arm_neon_vld4dup:
19101 case Intrinsic::arm_neon_vst1:
19102 case Intrinsic::arm_neon_vst1x2:
19103 case Intrinsic::arm_neon_vst1x3:
19104 case Intrinsic::arm_neon_vst1x4:
19105 case Intrinsic::arm_neon_vst2:
19106 case Intrinsic::arm_neon_vst3:
19107 case Intrinsic::arm_neon_vst4:
19108 case Intrinsic::arm_neon_vst2lane:
19109 case Intrinsic::arm_neon_vst3lane:
19110 case Intrinsic::arm_neon_vst4lane:
19112 case Intrinsic::arm_mve_vld2q:
19113 case Intrinsic::arm_mve_vld4q:
19114 case Intrinsic::arm_mve_vst2q:
19115 case Intrinsic::arm_mve_vst4q:
19126 return (VT == MVT::f32) && (Opc == ISD::LOAD || Opc == ISD::STORE);
19132 unsigned *
Fast)
const {
19141 if (Ty == MVT::i8 || Ty == MVT::i16 || Ty == MVT::i32) {
19143 if (AllowsUnaligned) {
19145 *
Fast = Subtarget->hasV7Ops();
19150 if (Ty == MVT::f64 || Ty == MVT::v2f64) {
19154 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->
isLittle())) {
19161 if (!Subtarget->hasMVEIntegerOps())
19165 if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1 ||
19166 Ty == MVT::v2i1)) {
19174 if ((Ty == MVT::v4i8 || Ty == MVT::v8i8 || Ty == MVT::v4i16) &&
19190 if (Ty == MVT::v16i8 || Ty == MVT::v8i16 || Ty == MVT::v8f16 ||
19191 Ty == MVT::v4i32 || Ty == MVT::v4f32 || Ty == MVT::v2i64 ||
19192 Ty == MVT::v2f64) {
19205 if ((
Op.isMemcpy() ||
Op.isZeroMemset()) && Subtarget->hasNEON() &&
19206 !FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat)) {
19208 if (
Op.size() >= 16 &&
19214 }
else if (
Op.size() >= 8 &&
19231 if (!SrcTy->isIntegerTy() || !DstTy->
isIntegerTy())
19233 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
19235 return (SrcBits == 64 && DestBits == 32);
19244 return (SrcBits == 64 && DestBits == 32);
19280 return Subtarget->hasFullFP16();
19290 return Ext->getType()->getScalarSizeInBits() ==
19291 2 * Ext->getOperand(0)->getType()->getScalarSizeInBits();
19308 if (!
I->getType()->isVectorTy())
19311 if (Subtarget->hasNEON()) {
19312 switch (
I->getOpcode()) {
19313 case Instruction::Sub:
19314 case Instruction::Add: {
19326 if (!Subtarget->hasMVEIntegerOps())
19330 if (!
I->hasOneUse())
19333 return Sub->getOpcode() == Instruction::FSub && Sub->getOperand(1) ==
I;
19343 switch (
I->getOpcode()) {
19344 case Instruction::Add:
19345 case Instruction::Mul:
19346 case Instruction::FAdd:
19347 case Instruction::ICmp:
19348 case Instruction::FCmp:
19350 case Instruction::FMul:
19351 return !IsFMSMul(
I);
19352 case Instruction::Sub:
19353 case Instruction::FSub:
19354 case Instruction::Shl:
19355 case Instruction::LShr:
19356 case Instruction::AShr:
19357 return Operand == 1;
19358 case Instruction::Call:
19360 switch (
II->getIntrinsicID()) {
19361 case Intrinsic::fma:
19363 case Intrinsic::sadd_sat:
19364 case Intrinsic::uadd_sat:
19365 case Intrinsic::arm_mve_add_predicated:
19366 case Intrinsic::arm_mve_mul_predicated:
19367 case Intrinsic::arm_mve_qadd_predicated:
19368 case Intrinsic::arm_mve_vhadd:
19369 case Intrinsic::arm_mve_hadd_predicated:
19370 case Intrinsic::arm_mve_vqdmull:
19371 case Intrinsic::arm_mve_vqdmull_predicated:
19372 case Intrinsic::arm_mve_vqdmulh:
19373 case Intrinsic::arm_mve_qdmulh_predicated:
19374 case Intrinsic::arm_mve_vqrdmulh:
19375 case Intrinsic::arm_mve_qrdmulh_predicated:
19376 case Intrinsic::arm_mve_fma_predicated:
19378 case Intrinsic::ssub_sat:
19379 case Intrinsic::usub_sat:
19380 case Intrinsic::arm_mve_sub_predicated:
19381 case Intrinsic::arm_mve_qsub_predicated:
19382 case Intrinsic::arm_mve_hsub_predicated:
19383 case Intrinsic::arm_mve_vhsub:
19384 return Operand == 1;
19395 for (
auto OpIdx :
enumerate(
I->operands())) {
19398 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
19402 if (Shuffle->
getOpcode() == Instruction::BitCast)
19410 if (!IsSinker(
I, OpIdx.index()))
19415 for (
Use &U :
Op->uses()) {
19417 if (!IsSinker(
Insn, U.getOperandNo()))
19430 if (!Subtarget->hasMVEIntegerOps())
19449 if (Ld->isExpandingLoad())
19453 if (Subtarget->hasMVEIntegerOps())
19498bool ARMTargetLowering::isFMAFasterThanFMulAndFAdd(
const MachineFunction &MF,
19506 return Subtarget->hasMVEFloatOps();
19524 unsigned Scale = 1;
19541 if ((V & (Scale - 1)) != 0)
19550 if (VT.
isVector() && Subtarget->hasNEON())
19553 !Subtarget->hasMVEFloatOps())
19556 bool IsNeg =
false;
19562 unsigned NumBytes = std::max((
unsigned)VT.
getSizeInBits() / 8, 1U);
19565 if (VT.
isVector() && Subtarget->hasMVEIntegerOps()) {
19581 if (VT.
isFloatingPoint() && NumBytes == 2 && Subtarget->hasFPRegs16())
19587 if (NumBytes == 1 || NumBytes == 2 || NumBytes == 4) {
19617 default:
return false;
19636 int Scale = AM.
Scale;
19641 default:
return false;
19649 Scale = Scale & ~1;
19650 return Scale == 2 || Scale == 4 || Scale == 8;
19667 if (Scale & 1)
return false;
19674 const int Scale = AM.
Scale;
19684 return (Scale == 1) || (!AM.
HasBaseReg && Scale == 2);
19700 switch (AM.
Scale) {
19717 int Scale = AM.
Scale;
19719 default:
return false;
19723 if (Scale < 0) Scale = -Scale;
19731 if (Scale == 1 || (AM.
HasBaseReg && Scale == -1))
19744 if (Scale & 1)
return false;
19757 if (!Subtarget->isThumb())
19764 return Imm >= 0 && Imm <= 255;
19773 int64_t AbsImm = std::abs(Imm);
19774 if (!Subtarget->isThumb())
19779 return AbsImm >= 0 && AbsImm <= 255;
19814 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) {
19818 int RHSC = (int)RHS->getZExtValue();
19819 if (RHSC < 0 && RHSC > -256) {
19829 }
else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) {
19832 int RHSC = (int)RHS->getZExtValue();
19833 if (RHSC < 0 && RHSC > -0x1000) {
19875 int RHSC = (int)RHS->getZExtValue();
19876 if (RHSC < 0 && RHSC > -0x100) {
19881 }
else if (RHSC > 0 && RHSC < 0x100) {
19892 bool isSEXTLoad,
bool IsMasked,
bool isLE,
19903 bool CanChangeType = isLE && !IsMasked;
19906 int RHSC = (int)RHS->getZExtValue();
19908 auto IsInRange = [&](
int RHSC,
int Limit,
int Scale) {
19909 if (RHSC < 0 && RHSC > -Limit * Scale && RHSC % Scale == 0) {
19914 }
else if (RHSC > 0 && RHSC < Limit * Scale && RHSC % Scale == 0) {
19925 if (VT == MVT::v4i16) {
19926 if (Alignment >= 2 && IsInRange(RHSC, 0x80, 2))
19928 }
else if (VT == MVT::v4i8 || VT == MVT::v8i8) {
19929 if (IsInRange(RHSC, 0x80, 1))
19931 }
else if (Alignment >= 4 &&
19932 (CanChangeType || VT == MVT::v4i32 || VT == MVT::v4f32) &&
19933 IsInRange(RHSC, 0x80, 4))
19935 else if (Alignment >= 2 &&
19936 (CanChangeType || VT == MVT::v8i16 || VT == MVT::v8f16) &&
19937 IsInRange(RHSC, 0x80, 2))
19939 else if ((CanChangeType || VT == MVT::v16i8) && IsInRange(RHSC, 0x80, 1))
19958 bool isSEXTLoad =
false;
19959 bool IsMasked =
false;
19961 Ptr = LD->getBasePtr();
19962 VT = LD->getMemoryVT();
19963 Alignment = LD->getAlign();
19966 Ptr = ST->getBasePtr();
19967 VT = ST->getMemoryVT();
19968 Alignment = ST->getAlign();
19970 Ptr = LD->getBasePtr();
19971 VT = LD->getMemoryVT();
19972 Alignment = LD->getAlign();
19976 Ptr = ST->getBasePtr();
19977 VT = ST->getMemoryVT();
19978 Alignment = ST->getAlign();
19984 bool isLegal =
false;
19986 isLegal = Subtarget->hasMVEIntegerOps() &&
19988 Ptr.getNode(), VT, Alignment, isSEXTLoad, IsMasked,
20016 bool isSEXTLoad =
false, isNonExt;
20017 bool IsMasked =
false;
20019 VT = LD->getMemoryVT();
20020 Ptr = LD->getBasePtr();
20021 Alignment = LD->getAlign();
20025 VT = ST->getMemoryVT();
20026 Ptr = ST->getBasePtr();
20027 Alignment = ST->getAlign();
20028 isNonExt = !ST->isTruncatingStore();
20030 VT = LD->getMemoryVT();
20031 Ptr = LD->getBasePtr();
20032 Alignment = LD->getAlign();
20037 VT = ST->getMemoryVT();
20038 Ptr = ST->getBasePtr();
20039 Alignment = ST->getAlign();
20040 isNonExt = !ST->isTruncatingStore();
20048 assert(
Op->getValueType(0) == MVT::i32 &&
"Non-i32 post-inc op?!");
20049 if (
Op->getOpcode() !=
ISD::ADD || !isNonExt)
20052 if (!RHS || RHS->getZExtValue() != 4)
20054 if (Alignment <
Align(4))
20058 Base =
Op->getOperand(0);
20064 bool isLegal =
false;
20066 isLegal = Subtarget->hasMVEIntegerOps() &&
20099 const APInt &DemandedElts,
20101 unsigned Depth)
const {
20104 switch (
Op.getOpcode()) {
20111 if (
Op.getResNo() == 0) {
20137 case Intrinsic::arm_ldaex:
20138 case Intrinsic::arm_ldrex: {
20153 const APInt &Mask =
Op.getConstantOperandAPInt(2);
20154 Known.
Zero &= Mask;
20160 const SDValue &SrcSV =
Op.getOperand(0);
20166 "VGETLANE index out of bounds");
20171 EVT VT =
Op.getValueType();
20178 Known = Known.
sext(DstSz);
20180 Known = Known.
zext(DstSz);
20188 Known = KnownOp.
zext(32);
20228 EVT VT =
Op.getValueType();
20234 assert(VT == MVT::i32 &&
"Unexpected integer type");
20241 unsigned Mask =
C->getZExtValue();
20244 unsigned ShrunkMask = Mask & Demanded;
20245 unsigned ExpandedMask = Mask | ~Demanded;
20249 if (ShrunkMask == 0)
20255 if (ExpandedMask == ~0U)
20258 auto IsLegalMask = [ShrunkMask, ExpandedMask](
unsigned Mask) ->
bool {
20259 return (ShrunkMask & Mask) == ShrunkMask && (~ExpandedMask & Mask) == 0;
20261 auto UseMask = [Mask,
Op, VT, &TLO](
unsigned NewMask) ->
bool {
20262 if (NewMask == Mask)
20271 if (IsLegalMask(0xFF))
20272 return UseMask(0xFF);
20275 if (IsLegalMask(0xFFFF))
20276 return UseMask(0xFFFF);
20280 if (ShrunkMask < 256)
20281 return UseMask(ShrunkMask);
20285 if ((
int)ExpandedMask <= -2 && (
int)ExpandedMask >= -256)
20286 return UseMask(ExpandedMask);
20301 unsigned Depth)
const {
20302 unsigned Opc =
Op.getOpcode();
20310 if (
Op.getResNo() == 0 && !
Op->hasAnyUseOfValue(1) &&
20312 unsigned ShAmt =
Op->getConstantOperandVal(2);
20324 unsigned ModImm =
Op.getConstantOperandVal(1);
20325 unsigned EltBits = 0;
20327 if ((OriginalDemandedBits & Mask) == 0)
20333 Op, OriginalDemandedBits, OriginalDemandedElts, Known, TLO,
Depth);
20342 if (!Subtarget->hasV6Ops())
20348 SplitString(AsmStr, AsmPieces,
";\n");
20350 switch (AsmPieces.
size()) {
20351 default:
return false;
20353 AsmStr = AsmPieces[0];
20355 SplitString(AsmStr, AsmPieces,
" \t,");
20358 if (AsmPieces.
size() == 3 &&
20359 AsmPieces[0] ==
"rev" && AsmPieces[1] ==
"$0" && AsmPieces[2] ==
"$1" &&
20360 IA->getConstraintString().compare(0, 4,
"=l,l") == 0) {
20362 if (Ty && Ty->getBitWidth() == 32)
20383 if (ConstraintVT.
isVector() && Subtarget->hasNEON() &&
20395 unsigned S = Constraint.
size();
20397 switch (Constraint[0]) {
20409 }
else if (S == 2) {
20410 switch (Constraint[0]) {
20427 Value *CallOperandVal =
info.CallOperandVal;
20430 if (!CallOperandVal)
20434 switch (*constraint) {
20439 if (type->isIntegerTy()) {
20440 if (Subtarget->isThumb())
20447 if (type->isFloatingPointTy())
20454using RCPair = std::pair<unsigned, const TargetRegisterClass *>;
20458 switch (Constraint.
size()) {
20461 switch (Constraint[0]) {
20463 if (Subtarget->isThumb())
20464 return RCPair(0U, &ARM::tGPRRegClass);
20465 return RCPair(0U, &ARM::GPRRegClass);
20467 if (Subtarget->isThumb())
20468 return RCPair(0U, &ARM::hGPRRegClass);
20472 return RCPair(0U, &ARM::tGPRRegClass);
20473 return RCPair(0U, &ARM::GPRRegClass);
20475 if (VT == MVT::Other)
20477 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20478 return RCPair(0U, &ARM::SPRRegClass);
20480 return RCPair(0U, &ARM::DPRRegClass);
20482 return RCPair(0U, &ARM::QPRRegClass);
20485 if (VT == MVT::Other)
20487 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20488 return RCPair(0U, &ARM::SPR_8RegClass);
20490 return RCPair(0U, &ARM::DPR_8RegClass);
20492 return RCPair(0U, &ARM::QPR_8RegClass);
20495 if (VT == MVT::Other)
20497 if (VT == MVT::f32 || VT == MVT::i32 || VT == MVT::f16 || VT == MVT::bf16)
20498 return RCPair(0U, &ARM::SPRRegClass);
20500 return RCPair(0U, &ARM::DPR_VFP2RegClass);
20502 return RCPair(0U, &ARM::QPR_VFP2RegClass);
20508 if (Constraint[0] ==
'T') {
20509 switch (Constraint[1]) {
20513 return RCPair(0U, &ARM::tGPREvenRegClass);
20515 return RCPair(0U, &ARM::tGPROddRegClass);
20524 if (
StringRef(
"{cc}").equals_insensitive(Constraint))
20525 return std::make_pair(
unsigned(ARM::CPSR), &ARM::CCRRegClass);
20534 std::vector<SDValue> &Ops,
20539 if (Constraint.
size() != 1)
20542 char ConstraintLetter = Constraint[0];
20543 switch (ConstraintLetter) {
20546 case 'I':
case 'J':
case 'K':
case 'L':
20547 case 'M':
case 'N':
case 'O':
20552 int64_t CVal64 =
C->getSExtValue();
20553 int CVal = (int) CVal64;
20556 if (CVal != CVal64)
20559 switch (ConstraintLetter) {
20563 if (Subtarget->hasV6T2Ops() || (Subtarget->hasV8MBaselineOps()))
20564 if (CVal >= 0 && CVal <= 65535)
20571 if (CVal >= 0 && CVal <= 255)
20573 }
else if (Subtarget->
isThumb2()) {
20592 if (CVal >= -255 && CVal <= -1)
20598 if (CVal >= -4095 && CVal <= 4095)
20611 }
else if (Subtarget->
isThumb2()) {
20634 if (CVal >= -7 && CVal < 7)
20636 }
else if (Subtarget->
isThumb2()) {
20659 if ((CVal >= 0 && CVal <= 1020) && ((CVal & 3) == 0))
20665 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0))
20673 if (CVal >= 0 && CVal <= 31)
20682 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0))
20692 if (Result.getNode()) {
20693 Ops.push_back(Result);
20703 "Unhandled Opcode in getDivRemLibcall");
20709 case MVT::i8: LC =
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
20710 case MVT::i16: LC =
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
20711 case MVT::i32: LC =
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
20712 case MVT::i64: LC =
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
20721 "Unhandled Opcode in getDivRemArgList");
20726 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
20727 EVT ArgVT =
N->getOperand(i).getValueType();
20729 Entry.Node =
N->getOperand(i);
20733 Args.push_back(Entry);
20744 "Register-based DivRem lowering only");
20745 unsigned Opcode =
Op->getOpcode();
20747 "Invalid opcode for Div/Rem lowering");
20749 EVT VT =
Op->getValueType(0);
20771 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
20772 : Subtarget->hasDivideInARMMode();
20773 if (hasDivide &&
Op->getValueType(0).isSimple() &&
20774 Op->getSimpleValueType(0) == MVT::i32) {
20776 const SDValue Dividend =
Op->getOperand(0);
20777 const SDValue Divisor =
Op->getOperand(1);
20778 SDValue Div = DAG.
getNode(DivOpcode, dl, VT, Dividend, Divisor);
20782 SDValue Values[2] = {Div, Rem};
20807 std::pair<SDValue, SDValue> CallInfo =
LowerCallTo(CLI);
20808 return CallInfo.first;
20814 EVT VT =
N->getValueType(0);
20820 Result[0], Result[1]);
20824 std::vector<Type*> RetTyParams;
20825 Type *RetTyElement;
20835 RetTyParams.push_back(RetTyElement);
20836 RetTyParams.push_back(RetTyElement);
20853 CallLoweringInfo CLI(DAG);
20857 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
20860 SDNode *ResNode = CallResult.first.getNode();
20875 "no-stack-arg-probe")) {
20886 SDValue Ops[2] = { SP, Chain };
20903 SDValue Ops[2] = { NewSP, Chain };
20908 bool IsStrict =
Op->isStrictFPOpcode();
20909 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
20910 const unsigned DstSz =
Op.getValueType().getSizeInBits();
20912 assert(DstSz > SrcSz && DstSz <= 64 && SrcSz >= 16 &&
20913 "Unexpected type for custom-lowering FP_EXTEND");
20916 "With both FP DP and 16, any FP conversion is legal!");
20918 assert(!(DstSz == 32 && Subtarget->hasFP16()) &&
20919 "With FP16, 16 to 32 conversion is legal!");
20922 if (SrcSz == 32 && DstSz == 64 && Subtarget->hasFP64()) {
20927 Loc,
Op.getValueType(), SrcVal);
20940 MakeLibCallOptions CallOptions;
20942 for (
unsigned Sz = SrcSz; Sz <= 32 && Sz < DstSz; Sz *= 2) {
20943 bool Supported = (Sz == 16 ? Subtarget->hasFP16() : Subtarget->hasFP64());
20944 MVT SrcVT = (Sz == 16 ? MVT::f16 : MVT::f32);
20945 MVT DstVT = (Sz == 16 ? MVT::f32 : MVT::f64);
20949 {DstVT, MVT::Other}, {Chain, SrcVal});
20952 SrcVal = DAG.
getNode(ISD::FP_EXTEND, Loc, DstVT, SrcVal);
20956 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20957 "Unexpected type for custom-lowering FP_EXTEND");
20958 std::tie(SrcVal, Chain) =
makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
20963 return IsStrict ? DAG.
getMergeValues({SrcVal, Chain}, Loc) : SrcVal;
20967 bool IsStrict =
Op->isStrictFPOpcode();
20969 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
20971 EVT DstVT =
Op.getValueType();
20972 const unsigned DstSz =
Op.getValueType().getSizeInBits();
20975 assert(DstSz < SrcSz && SrcSz <= 64 && DstSz >= 16 &&
20976 "Unexpected type for custom-lowering FP_ROUND");
20979 "With both FP DP and 16, any FP conversion is legal!");
20984 if (SrcSz == 32 && Subtarget->hasFP16())
20989 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20990 "Unexpected type for custom-lowering FP_ROUND");
20991 MakeLibCallOptions CallOptions;
20994 std::tie(Result, Chain) =
makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
21006 if (v == 0xffffffff)
21018 bool ForCodeSize)
const {
21021 if (VT == MVT::f16 && Subtarget->hasFullFP16())
21023 if (VT == MVT::f32 && Subtarget->hasFullFP16() &&
21026 if (VT == MVT::f32)
21028 if (VT == MVT::f64 && Subtarget->hasFP64())
21039 unsigned Intrinsic)
const {
21040 switch (Intrinsic) {
21041 case Intrinsic::arm_neon_vld1:
21042 case Intrinsic::arm_neon_vld2:
21043 case Intrinsic::arm_neon_vld3:
21044 case Intrinsic::arm_neon_vld4:
21045 case Intrinsic::arm_neon_vld2lane:
21046 case Intrinsic::arm_neon_vld3lane:
21047 case Intrinsic::arm_neon_vld4lane:
21048 case Intrinsic::arm_neon_vld2dup:
21049 case Intrinsic::arm_neon_vld3dup:
21050 case Intrinsic::arm_neon_vld4dup: {
21053 auto &
DL =
I.getDataLayout();
21054 uint64_t NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
21056 Info.ptrVal =
I.getArgOperand(0);
21058 Value *AlignArg =
I.getArgOperand(
I.arg_size() - 1);
21064 case Intrinsic::arm_neon_vld1x2:
21065 case Intrinsic::arm_neon_vld1x3:
21066 case Intrinsic::arm_neon_vld1x4: {
21069 auto &
DL =
I.getDataLayout();
21070 uint64_t NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
21072 Info.ptrVal =
I.getArgOperand(
I.arg_size() - 1);
21074 Info.align =
I.getParamAlign(
I.arg_size() - 1).valueOrOne();
21079 case Intrinsic::arm_neon_vst1:
21080 case Intrinsic::arm_neon_vst2:
21081 case Intrinsic::arm_neon_vst3:
21082 case Intrinsic::arm_neon_vst4:
21083 case Intrinsic::arm_neon_vst2lane:
21084 case Intrinsic::arm_neon_vst3lane:
21085 case Intrinsic::arm_neon_vst4lane: {
21088 auto &
DL =
I.getDataLayout();
21089 unsigned NumElts = 0;
21090 for (
unsigned ArgI = 1, ArgE =
I.arg_size(); ArgI < ArgE; ++ArgI) {
21091 Type *ArgTy =
I.getArgOperand(ArgI)->getType();
21094 NumElts +=
DL.getTypeSizeInBits(ArgTy) / 64;
21097 Info.ptrVal =
I.getArgOperand(0);
21099 Value *AlignArg =
I.getArgOperand(
I.arg_size() - 1);
21105 case Intrinsic::arm_neon_vst1x2:
21106 case Intrinsic::arm_neon_vst1x3:
21107 case Intrinsic::arm_neon_vst1x4: {
21110 auto &
DL =
I.getDataLayout();
21111 unsigned NumElts = 0;
21112 for (
unsigned ArgI = 1, ArgE =
I.arg_size(); ArgI < ArgE; ++ArgI) {
21113 Type *ArgTy =
I.getArgOperand(ArgI)->getType();
21116 NumElts +=
DL.getTypeSizeInBits(ArgTy) / 64;
21119 Info.ptrVal =
I.getArgOperand(0);
21121 Info.align =
I.getParamAlign(0).valueOrOne();
21126 case Intrinsic::arm_mve_vld2q:
21127 case Intrinsic::arm_mve_vld4q: {
21131 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vld2q ? 2 : 4;
21133 Info.ptrVal =
I.getArgOperand(0);
21140 case Intrinsic::arm_mve_vst2q:
21141 case Intrinsic::arm_mve_vst4q: {
21144 Type *VecTy =
I.getArgOperand(1)->getType();
21145 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vst2q ? 2 : 4;
21147 Info.ptrVal =
I.getArgOperand(0);
21154 case Intrinsic::arm_mve_vldr_gather_base:
21155 case Intrinsic::arm_mve_vldr_gather_base_predicated: {
21157 Info.ptrVal =
nullptr;
21159 Info.align =
Align(1);
21163 case Intrinsic::arm_mve_vldr_gather_base_wb:
21164 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
21166 Info.ptrVal =
nullptr;
21167 Info.memVT =
MVT::getVT(
I.getType()->getContainedType(0));
21168 Info.align =
Align(1);
21172 case Intrinsic::arm_mve_vldr_gather_offset:
21173 case Intrinsic::arm_mve_vldr_gather_offset_predicated: {
21175 Info.ptrVal =
nullptr;
21180 Info.align =
Align(1);
21184 case Intrinsic::arm_mve_vstr_scatter_base:
21185 case Intrinsic::arm_mve_vstr_scatter_base_predicated: {
21187 Info.ptrVal =
nullptr;
21188 Info.memVT =
MVT::getVT(
I.getArgOperand(2)->getType());
21189 Info.align =
Align(1);
21193 case Intrinsic::arm_mve_vstr_scatter_base_wb:
21194 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated: {
21196 Info.ptrVal =
nullptr;
21197 Info.memVT =
MVT::getVT(
I.getArgOperand(2)->getType());
21198 Info.align =
Align(1);
21202 case Intrinsic::arm_mve_vstr_scatter_offset:
21203 case Intrinsic::arm_mve_vstr_scatter_offset_predicated: {
21205 Info.ptrVal =
nullptr;
21210 Info.align =
Align(1);
21214 case Intrinsic::arm_ldaex:
21215 case Intrinsic::arm_ldrex: {
21216 auto &
DL =
I.getDataLayout();
21217 Type *ValTy =
I.getParamElementType(0);
21220 Info.ptrVal =
I.getArgOperand(0);
21222 Info.align =
DL.getABITypeAlign(ValTy);
21226 case Intrinsic::arm_stlex:
21227 case Intrinsic::arm_strex: {
21228 auto &
DL =
I.getDataLayout();
21229 Type *ValTy =
I.getParamElementType(1);
21232 Info.ptrVal =
I.getArgOperand(1);
21234 Info.align =
DL.getABITypeAlign(ValTy);
21238 case Intrinsic::arm_stlexd:
21239 case Intrinsic::arm_strexd:
21241 Info.memVT = MVT::i64;
21242 Info.ptrVal =
I.getArgOperand(2);
21244 Info.align =
Align(8);
21248 case Intrinsic::arm_ldaexd:
21249 case Intrinsic::arm_ldrexd:
21251 Info.memVT = MVT::i64;
21252 Info.ptrVal =
I.getArgOperand(0);
21254 Info.align =
Align(8);
21269 assert(Ty->isIntegerTy());
21271 unsigned Bits = Ty->getPrimitiveSizeInBits();
21272 if (Bits == 0 || Bits > 32)
21278 unsigned Index)
const {
21287 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
21290 if (!Subtarget->hasDataBarrier()) {
21294 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) {
21296 Value*
args[6] = {Builder.getInt32(15), Builder.getInt32(0),
21297 Builder.getInt32(0), Builder.getInt32(7),
21298 Builder.getInt32(10), Builder.getInt32(5)};
21299 return Builder.CreateCall(MCR,
args);
21310 return Builder.CreateCall(DMB, CDomain);
21331 if (Subtarget->preferISHSTBarriers())
21364 bool has64BitAtomicStore;
21366 has64BitAtomicStore =
false;
21367 else if (Subtarget->isThumb())
21368 has64BitAtomicStore = Subtarget->hasV7Ops();
21370 has64BitAtomicStore = Subtarget->hasV6Ops();
21372 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
21386 bool has64BitAtomicLoad;
21388 has64BitAtomicLoad =
false;
21389 else if (Subtarget->isThumb())
21390 has64BitAtomicLoad = Subtarget->hasV7Ops();
21392 has64BitAtomicLoad = Subtarget->hasV6Ops();
21409 hasAtomicRMW = Subtarget->hasV8MBaselineOps();
21410 else if (Subtarget->isThumb())
21411 hasAtomicRMW = Subtarget->hasV7Ops();
21413 hasAtomicRMW = Subtarget->hasV6Ops();
21414 if (
Size <= (Subtarget->
isMClass() ? 32U : 64U) && hasAtomicRMW) {
21437 bool HasAtomicCmpXchg;
21439 HasAtomicCmpXchg = Subtarget->hasV8MBaselineOps();
21440 else if (Subtarget->isThumb())
21441 HasAtomicCmpXchg = Subtarget->hasV7Ops();
21443 HasAtomicCmpXchg = Subtarget->hasV6Ops();
21445 HasAtomicCmpXchg &&
Size <= (Subtarget->
isMClass() ? 32U : 64U))
21452 return InsertFencesForAtomic;
21465 M.getOrInsertGlobal(
"__security_cookie",
21473 F->addParamAttr(0, Attribute::AttrKind::InReg);
21479 return M.getGlobalVariable(
"__security_cookie");
21491 unsigned &
Cost)
const {
21493 if (!Subtarget->hasNEON())
21520 return Subtarget->hasV6T2Ops();
21524 return Subtarget->hasV6T2Ops();
21529 if (!Subtarget->hasV7Ops())
21535 if (!Mask || Mask->getValue().getBitWidth() > 32u)
21537 auto MaskVal =
unsigned(Mask->getValue().getZExtValue());
21554 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
21560 if (ValueTy->getPrimitiveSizeInBits() == 64) {
21562 IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd;
21565 Value *LoHi = Builder.CreateCall(Ldrex,
Addr,
"lohi");
21567 Value *
Lo = Builder.CreateExtractValue(LoHi, 0,
"lo");
21568 Value *
Hi = Builder.CreateExtractValue(LoHi, 1,
"hi");
21571 Lo = Builder.CreateZExt(
Lo, ValueTy,
"lo64");
21572 Hi = Builder.CreateZExt(
Hi, ValueTy,
"hi64");
21573 return Builder.CreateOr(
21574 Lo, Builder.CreateShl(
Hi, ConstantInt::get(ValueTy, 32)),
"val64");
21577 Type *Tys[] = {
Addr->getType() };
21578 Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
21583 0,
Attribute::get(M->getContext(), Attribute::ElementType, ValueTy));
21584 return Builder.CreateTruncOrBitCast(CI, ValueTy);
21589 if (!Subtarget->hasV7Ops())
21591 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
21598 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
21606 IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd;
21611 Value *
Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 32),
Int32Ty,
"hi");
21614 return Builder.CreateCall(Strex, {
Lo,
Hi,
Addr});
21617 Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex;
21618 Type *Tys[] = {
Addr->getType() };
21621 CallInst *CI = Builder.CreateCall(
21622 Strex, {Builder.CreateZExtOrBitCast(
21640 return (
DL.getTypeSizeInBits(VecTy) + 127) / 128;
21647 unsigned VecSize =
DL.getTypeSizeInBits(VecTy);
21650 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps())
21658 if (Subtarget->hasMVEIntegerOps() && Factor == 3)
21666 if (ElSize != 8 && ElSize != 16 && ElSize != 32)
21669 if (Subtarget->hasMVEIntegerOps() && Alignment < ElSize / 8)
21674 if (Subtarget->hasNEON() && VecSize == 64)
21676 return VecSize % 128 == 0;
21680 if (Subtarget->hasNEON())
21682 if (Subtarget->hasMVEIntegerOps())
21702 "Invalid interleave factor");
21703 assert(!Shuffles.
empty() &&
"Empty shufflevector input");
21705 "Unmatched number of shufflevectors and indices");
21708 Type *EltTy = VecTy->getElementType();
21731 if (NumLoads > 1) {
21735 VecTy->getNumElements() / NumLoads);
21740 auto createLoadIntrinsic = [&](
Value *BaseAddr) {
21741 if (Subtarget->hasNEON()) {
21743 Type *Tys[] = {VecTy, PtrTy};
21744 static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2,
21745 Intrinsic::arm_neon_vld3,
21746 Intrinsic::arm_neon_vld4};
21754 return Builder.CreateCall(VldnFunc, Ops,
"vldN");
21756 assert((Factor == 2 || Factor == 4) &&
21757 "expected interleave factor of 2 or 4 for MVE");
21759 Factor == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
21761 Type *Tys[] = {VecTy, PtrTy};
21767 return Builder.CreateCall(VldnFunc, Ops,
"vldN");
21776 for (
unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) {
21780 BaseAddr = Builder.CreateConstGEP1_32(VecTy->getElementType(), BaseAddr,
21781 VecTy->getNumElements() * Factor);
21783 CallInst *VldN = createLoadIntrinsic(BaseAddr);
21787 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
21789 unsigned Index = Indices[i];
21791 Value *SubVec = Builder.CreateExtractValue(VldN,
Index);
21795 SubVec = Builder.CreateIntToPtr(
21799 SubVecs[SV].push_back(SubVec);
21808 auto &SubVec = SubVecs[SVI];
21811 SVI->replaceAllUsesWith(WideVec);
21845 unsigned Factor)
const {
21847 "Invalid interleave factor");
21850 assert(VecTy->getNumElements() % Factor == 0 &&
"Invalid interleaved store");
21852 unsigned LaneLen = VecTy->getNumElements() / Factor;
21853 Type *EltTy = VecTy->getElementType();
21857 Align Alignment = SI->getAlign();
21874 Type *IntTy =
DL.getIntPtrType(EltTy);
21879 Op0 = Builder.CreatePtrToInt(Op0, IntVecTy);
21880 Op1 = Builder.CreatePtrToInt(Op1, IntVecTy);
21886 Value *BaseAddr = SI->getPointerOperand();
21888 if (NumStores > 1) {
21891 LaneLen /= NumStores;
21899 auto createStoreIntrinsic = [&](
Value *BaseAddr,
21901 if (Subtarget->hasNEON()) {
21902 static const Intrinsic::ID StoreInts[3] = {Intrinsic::arm_neon_vst2,
21903 Intrinsic::arm_neon_vst3,
21904 Intrinsic::arm_neon_vst4};
21905 Type *PtrTy = Builder.getPtrTy(SI->getPointerAddressSpace());
21906 Type *Tys[] = {PtrTy, SubVecTy};
21909 SI->getModule(), StoreInts[Factor - 2], Tys);
21914 Ops.
push_back(Builder.getInt32(SI->getAlign().value()));
21915 Builder.CreateCall(VstNFunc, Ops);
21917 assert((Factor == 2 || Factor == 4) &&
21918 "expected interleave factor of 2 or 4 for MVE");
21920 Factor == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
21921 Type *PtrTy = Builder.getPtrTy(SI->getPointerAddressSpace());
21922 Type *Tys[] = {PtrTy, SubVecTy};
21929 for (
unsigned F = 0;
F < Factor;
F++) {
21931 Builder.CreateCall(VstNFunc, Ops);
21937 for (
unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
21940 if (StoreCount > 0)
21941 BaseAddr = Builder.CreateConstGEP1_32(SubVecTy->getElementType(),
21942 BaseAddr, LaneLen * Factor);
21947 for (
unsigned i = 0; i < Factor; i++) {
21948 unsigned IdxI = StoreCount * LaneLen * Factor + i;
21949 if (Mask[IdxI] >= 0) {
21950 Shuffles.
push_back(Builder.CreateShuffleVector(
21953 unsigned StartMask = 0;
21954 for (
unsigned j = 1; j < LaneLen; j++) {
21955 unsigned IdxJ = StoreCount * LaneLen * Factor + j;
21956 if (Mask[IdxJ * Factor + IdxI] >= 0) {
21957 StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ;
21967 Shuffles.
push_back(Builder.CreateShuffleVector(
21972 createStoreIntrinsic(BaseAddr, Shuffles);
21988 for (
unsigned i = 0; i < ST->getNumElements(); ++i) {
21992 Members += SubMembers;
21998 Members += SubMembers * AT->getNumElements();
21999 }
else if (Ty->isFloatTy()) {
22004 }
else if (Ty->isDoubleTy()) {
22016 return VT->getPrimitiveSizeInBits().getFixedValue() == 64;
22018 return VT->getPrimitiveSizeInBits().getFixedValue() == 128;
22020 switch (VT->getPrimitiveSizeInBits().getFixedValue()) {
22033 return (Members > 0 && Members <= 4);
22039 const Align ABITypeAlign =
DL.getABITypeAlign(ArgTy);
22041 return ABITypeAlign;
22046 assert(StackAlign &&
"data layout string is missing stack alignment");
22047 return std::min(ABITypeAlign, *StackAlign);
22054 Type *Ty, CallingConv::ID CallConv,
bool isVarArg,
22056 if (getEffectiveCallingConv(CallConv, isVarArg) !=
22065 bool IsIntArray = Ty->isArrayTy() && Ty->getArrayElementType()->isIntegerTy();
22066 return IsHA || IsIntArray;
22070 const Constant *PersonalityFn)
const {
22077 const Constant *PersonalityFn)
const {
22089void ARMTargetLowering::insertCopiesSplitCSR(
22093 const MCPhysReg *IStart =
TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
22103 RC = &ARM::GPRRegClass;
22104 else if (ARM::DPRRegClass.
contains(*
I))
22105 RC = &ARM::DPRRegClass;
22115 assert(Entry->getParent()->getFunction().hasFnAttribute(
22116 Attribute::NoUnwind) &&
22117 "Function should be nounwind in insertCopiesSplitCSR!");
22118 Entry->addLiveIn(*
I);
22123 for (
auto *Exit : Exits)
22125 TII->get(TargetOpcode::COPY), *
I)
22136 return Subtarget->hasMVEIntegerOps();
22146 unsigned NumElements = VTy->getNumElements();
22153 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy())
22154 return Subtarget->hasMVEFloatOps();
22159 return Subtarget->hasMVEIntegerOps() &&
22160 (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
22161 ScalarTy->isIntegerTy(32));
22171 unsigned TyWidth = Ty->getScalarSizeInBits() * Ty->getNumElements();
22173 assert(TyWidth >= 128 &&
"Width of vector type must be at least 128 bits");
22175 if (TyWidth > 128) {
22176 int Stride = Ty->getNumElements() / 2;
22180 ArrayRef<int> UpperSplitMask(&SplitSeqVec[Stride], Stride);
22182 auto *LowerSplitA =
B.CreateShuffleVector(InputA, LowerSplitMask);
22183 auto *LowerSplitB =
B.CreateShuffleVector(InputB, LowerSplitMask);
22184 auto *UpperSplitA =
B.CreateShuffleVector(InputA, UpperSplitMask);
22185 auto *UpperSplitB =
B.CreateShuffleVector(InputB, UpperSplitMask);
22186 Value *LowerSplitAcc =
nullptr;
22187 Value *UpperSplitAcc =
nullptr;
22190 LowerSplitAcc =
B.CreateShuffleVector(
Accumulator, LowerSplitMask);
22191 UpperSplitAcc =
B.CreateShuffleVector(
Accumulator, UpperSplitMask);
22195 B, OperationType, Rotation, LowerSplitA, LowerSplitB, LowerSplitAcc);
22197 B, OperationType, Rotation, UpperSplitA, UpperSplitB, UpperSplitAcc);
22199 ArrayRef<int> JoinMask(&SplitSeqVec[0], Ty->getNumElements());
22200 return B.CreateShuffleVector(LowerSplitInt, UpperSplitInt, JoinMask);
22207 ConstRotation = ConstantInt::get(IntTy, (
int)Rotation);
22210 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmlaq, Ty,
22212 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmulq, Ty,
22213 {ConstRotation, InputB, InputA});
22218 auto *ConstHalving = ConstantInt::get(IntTy, 1);
22221 ConstRotation = ConstantInt::get(IntTy, 0);
22223 ConstRotation = ConstantInt::get(IntTy, 1);
22225 if (!ConstRotation)
22228 return B.CreateIntrinsic(Intrinsic::arm_mve_vcaddq, Ty,
22229 {ConstHalving, ConstRotation, InputA, InputB});
unsigned const MachineRegisterInfo * MRI
static bool isAddSubSExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt)
isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift righ...
static bool isExtendedBUILD_VECTOR(SDValue N, SelectionDAG &DAG, bool isSigned)
static bool isZeroExtended(SDValue N, SelectionDAG &DAG)
static bool areExtractExts(Value *Ext1, Value *Ext2)
Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements.
static EVT getExtensionTo64Bits(const EVT &OrigVT)
static const MCPhysReg GPRArgRegs[]
static SDValue GeneratePerfectShuffle(unsigned ID, SDValue V1, SDValue V2, unsigned PFEntry, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl)
GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations t...
static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt)
getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift oper...
static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG)
static bool isSignExtended(SDValue N, SelectionDAG &DAG)
static bool isAddSubZExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt)
isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left...
static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls)
Return true if the calling convention is one that we can guarantee TCO for.
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG)
static bool isStore(int Opcode)
static bool isThumb(const MCSubtargetInfo &STI)
static SDValue LowerUADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG)
static SDValue PerformExtractEltToVMOVRRD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, MachineFrameInfo &MFI, const MachineRegisterInfo *MRI, const TargetInstrInfo *TII)
MatchingStackOffset - Return true if the given stack call argument is already available in the same p...
static SDValue PerformVQDMULHCombine(SDNode *N, SelectionDAG &DAG)
static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG, const EVT &OrigTy, const EVT &ExtTy, unsigned ExtOpcode)
AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total value size to 64 bits.
static cl::opt< unsigned > ConstpoolPromotionMaxSize("arm-promote-constant-max-size", cl::Hidden, cl::desc("Maximum size of constant to promote into a constant pool"), cl::init(64))
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue LowerINSERT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isVTBLMask(ArrayRef< int > M, EVT VT)
static SDValue PerformSUBCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
static cl::opt< bool > EnableConstpoolPromotion("arm-promote-constant", cl::Hidden, cl::desc("Enable / disable promotion of unnamed_addr constants into " "constant pools"), cl::init(false))
static SDValue PerformFAddVSelectCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformExtractFpToIntStores(StoreSDNode *St, SelectionDAG &DAG)
static SDValue PerformVDUPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP.
static SDValue PerformExtractEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static const APInt * isPowerOf2Constant(SDValue V)
static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD) can replace combinations of ...
static SDValue PerformVMOVhrCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG)
static SDValue LowerVECTOR_SHUFFLEUsingOneOff(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static bool isValidMVECond(unsigned CC, bool IsFloat)
static SDValue PerformPREDICATE_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC)
IntCCToARMCC - Convert a DAG integer condition code to an ARM CC.
static SDValue PerformSTORECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSTORECombine - Target-specific dag combine xforms for ISD::STORE.
static SDValue ConvertBooleanCarryToCarryFlag(SDValue BoolCarry, SelectionDAG &DAG)
static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isGTorGE(ISD::CondCode CC)
static bool CombineVLDDUP(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a vldN-lane (N > 1) intrinsic,...
static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask)
static bool isReverseMask(ArrayRef< int > M, EVT VT)
static bool isVZIP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of "vector_shuffle v,...
static SDValue PerformSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue AddCombineTo64bitUMAAL(SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVECTOR_REG_CASTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVMulVCTPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVMulVCTPCombine - VCVT (fixed-point to floating-point, Advanced SIMD) can replace combinations...
static SDValue createGPRPairNode2xi32(SelectionDAG &DAG, SDValue V0, SDValue V1)
static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG)
static bool findPointerConstIncrement(SDNode *N, SDValue *Ptr, SDValue *CInc)
static bool isVTRNMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool CanInvertMVEVCMP(SDValue N)
static SDValue PerformLongShiftCombine(SDNode *N, SelectionDAG &DAG)
static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformShiftCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
PerformShiftCombine - Checks for immediate versions of vector shifts and lowers them.
static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode, ARMCC::CondCodes &CondCode2)
FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static EVT getVectorTyFromPredicateVector(EVT VT)
static SDValue PerformFADDVCMLACombine(SDNode *N, SelectionDAG &DAG)
static SDValue handleCMSEValue(const SDValue &Value, const ISD::InputArg &Arg, SelectionDAG &DAG, const SDLoc &DL)
static SDValue PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
static bool isSRL16(const SDValue &Op)
static SDValue PerformVMOVrhCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformLOADCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue IsCMPZCSINC(SDNode *Cmp, ARMCC::CondCodes &CC)
static unsigned getPointerConstIncrement(unsigned Opcode, SDValue Ptr, SDValue Inc, const SelectionDAG &DAG)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static Register genTPEntry(MachineBasicBlock *TpEntry, MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpExit, Register OpSizeReg, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI)
Adds logic in loop entry MBB to calculate loop iteration count and adds t2WhileLoopSetup and t2WhileL...
static SDValue createGPRPairNodei64(SelectionDAG &DAG, SDValue V)
static bool isLTorLE(ISD::CondCode CC)
static SDValue PerformVCMPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformMVEVMULLCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl, SelectionDAG &DAG)
static SDValue PerformBITCASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue AddCombineTo64bitMLAL(SDNode *AddeSubeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG)
static bool checkAndUpdateCPSRKill(MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI)
static SDValue PerformCMPZCombine(SDNode *N, SelectionDAG &DAG)
static bool hasNormalLoadOperand(SDNode *N)
hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node are normal,...
static SDValue PerformInsertEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
PerformInsertEltCombine - Target-specific dag combine xforms for ISD::INSERT_VECTOR_ELT.
static SDValue PerformVDUPLANECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVDUPLANECombine - Target-specific dag combine xforms for ARMISD::VDUPLANE.
static SDValue LowerBuildVectorOfFPTrunc(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static cl::opt< unsigned > ConstpoolPromotionMaxTotal("arm-promote-constant-max-total", cl::Hidden, cl::desc("Maximum size of ALL constants to promote into a constant pool"), cl::init(128))
static SDValue LowerTruncatei1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static RTLIB::Libcall getDivRemLibcall(const SDNode *N, MVT::SimpleValueType SVT)
static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG &DAG)
SkipLoadExtensionForVMULL - return a load of the original vector size that does not do any sign/zero ...
static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static const MCPhysReg GPRArgRegs[]
static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombineWithOperands - Try DAG combinations for an ADD with operands N0 and N1.
static SDValue PromoteMVEPredVector(SDLoc dl, SDValue Pred, EVT VT, SelectionDAG &DAG)
static bool isVZIPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue PerformORCombineToSMULWBT(SDNode *OR, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool isVTRN_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of "vector_shuffle v,...
static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue FindBFIToCombineWith(SDNode *N)
static SDValue LowerADDSUBSAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT, SelectionDAG &DAG)
static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode, bool &swpCmpOps, bool &swpVselOps)
static void ReplaceLongIntrinsic(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isS16(const SDValue &Op, SelectionDAG &DAG)
static bool isSRA16(const SDValue &Op)
static SDValue AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerVECTOR_SHUFFLEUsingMovs(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue LowerInterruptReturn(SmallVectorImpl< SDValue > &RetOps, const SDLoc &DL, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl, SelectionDAG &DAG)
static void expandf64Toi32(SDValue Op, SelectionDAG &DAG, SDValue &RetVal1, SDValue &RetVal2)
static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isSHL16(const SDValue &Op)
static bool isVEXTMask(ArrayRef< int > M, EVT VT, bool &ReverseVEXT, unsigned &Imm)
static SDValue PerformMVEVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isTruncMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue PerformADDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2)
Return the load opcode for a given load size.
static bool isLegalT2AddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
static bool isLegalMVEShuffleOp(unsigned PFEntry)
static SDValue PerformSignExtendInregCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformShuffleVMOVNCombine(ShuffleVectorSDNode *N, SelectionDAG &DAG)
static bool isVUZPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG)
PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for ISD::VECTOR_SHUFFLE.
static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG)
SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND, ANY_EXTEND,...
static bool isVMOVNTruncMask(ArrayRef< int > M, EVT ToVT, bool rev)
static SDValue PerformVQMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static MachineBasicBlock * OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ)
static SDValue LowerVecReduceMinMax(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformFPExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformAddcSubcCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static TargetLowering::ArgListTy getDivRemArgList(const SDNode *N, LLVMContext *Context, const ARMSubtarget *Subtarget)
static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl)
getZeroVector - Returns a vector of specified type with all zero elements.
static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static ARMCC::CondCodes getVCMPCondCode(SDValue N)
static cl::opt< bool > ARMInterworking("arm-interworking", cl::Hidden, cl::desc("Enable / disable ARM interworking (for debugging only)"), cl::init(true))
static void ReplaceREADCYCLECOUNTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformORCombineToBFI(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes, SDValue &CC, bool &Invert, SDValue &OtherOp, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVSetCCToVCTPCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isZeroVector(SDValue N)
static SDValue PerformAddeSubeCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void ReplaceCMP_SWAP_64Results(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isLowerSaturate(const SDValue LHS, const SDValue RHS, const SDValue TrueVal, const SDValue FalseVal, const ISD::CondCode CC, const SDValue K)
static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG)
static void emitPostSt(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned StSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment store operation with given size.
static bool isVMOVNMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue CombineBaseUpdate(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineBaseUpdate - Target-specific DAG combine function for VLDDUP, NEON load/store intrinsics,...
static SDValue LowerSaturatingConditional(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSubCSINCCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVRRDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMOVRRDCombine - Target-specific dag combine xforms for ARMISD::VMOVRRD.
static SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformCSETCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue PerformInsertSubvectorCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorExtend(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain)
static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue PerformVMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMULCombine Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the special multi...
static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG)
static SDValue PerformBFICombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformORCombine - Target-specific dag combine xforms for ISD::OR.
static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG)
static SDValue PerformTruncatingStoreCombine(StoreSDNode *St, SelectionDAG &DAG)
static unsigned SelectPairHalf(unsigned Elements, ArrayRef< int > Mask, unsigned Index)
static void emitPostLd(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned LdSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment load operation with given size.
static SDValue TryDistrubutionADDVecReduce(SDNode *N, SelectionDAG &DAG)
static bool isValidBaseUpdate(SDNode *N, SDNode *User)
static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG, const ARMSubtarget *ST, const SDLoc &dl)
static bool IsQRMVEInstruction(const SDNode *N, const SDNode *Op)
static SDValue PerformMinMaxToSatCombine(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformXORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, Align Alignment, bool isSEXTLoad, bool IsMasked, bool isLE, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
std::pair< unsigned, const TargetRegisterClass * > RCPair
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND, ISD::ZERO_EXTEND,...
static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
cl::opt< unsigned > MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden, cl::desc("Maximum interleave factor for MVE VLDn to generate."), cl::init(2))
static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef, unsigned SplatBitSize, SelectionDAG &DAG, const SDLoc &dl, EVT &VT, EVT VectorVT, VMOVModImmType type)
isVMOVModifiedImm - Check if the specified splat value corresponds to a valid vector constant for a N...
static SDValue LowerBuildVectorOfFPExt(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC, SelectionDAG &DAG)
BC is a bitcast that is about to be turned into a VMOVDRR.
static SDValue promoteToConstantPool(const ARMTargetLowering *TLI, const GlobalValue *GV, SelectionDAG &DAG, EVT PtrVT, const SDLoc &dl)
static unsigned isNEONTwoResultShuffleMask(ArrayRef< int > ShuffleMask, EVT VT, unsigned &WhichResult, bool &isV_UNDEF)
Check if ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN), and return the corresponding AR...
static bool BitsProperlyConcatenate(const APInt &A, const APInt &B)
static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static SDValue LowerVecReduce(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG)
static bool TryCombineBaseUpdate(struct BaseUpdateTarget &Target, struct BaseUpdateUser &User, bool SimpleConstIncOnly, TargetLowering::DAGCombinerInfo &DCI)
static bool allUsersAreInFunction(const Value *V, const Function *F)
Return true if all users of V are within function F, looking through ConstantExprs.
static bool isSingletonVEXTMask(ArrayRef< int > M, EVT VT, unsigned &Imm)
static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG)
PerformVMOVDRRCombine - Target-specific dag combine xforms for ARMISD::VMOVDRR.
static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V, SDValue &SatK)
static bool isLegalAddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
isLegalAddressImmediate - Return true if the integer value can be used as the offset of the target ad...
static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isLegalT1AddressImmediate(int64_t V, EVT VT)
static SDValue CombineANDShift(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSHLSimplify(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformADDECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDECombine - Target-specific dag combine transform from ARMISD::ADDC, ARMISD::ADDE,...
static SDValue PerformReduceShuffleCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerTruncate(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformHWLoopCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformSplittingMVETruncToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool isVUZP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of "vector_shuffle v,...
static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base, uint64_t &Members)
static SDValue PerformMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerReverse_VECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
static SDValue PerformANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformADDVecReduce(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG)
static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm, bool &Negate)
static bool canChangeToInt(SDValue Op, bool &SeenZero, const ARMSubtarget *Subtarget)
canChangeToInt - Given the fp compare operand, return true if it is suitable to morph to an integer c...
static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2)
Return the store opcode for a given store size.
static bool IsVUZPShuffleNode(SDNode *N)
static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void attachMEMCPYScratchRegs(const ARMSubtarget *Subtarget, MachineInstr &MI, const SDNode *Node)
Attaches vregs to MEMCPY that it will use as scratch registers when it is expanded into LDM/STM.
static bool isFloatingPointZero(SDValue Op)
isFloatingPointZero - Return true if this is +0.0.
static SDValue findMUL_LOHI(SDValue V)
static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformORCombine_i1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformSplittingMVEEXTToWideningLoad(SDNode *N, SelectionDAG &DAG)
static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG)
static void genTPLoopBody(MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpEntry, MachineBasicBlock *TpExit, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI, Register OpSrcReg, Register OpDestReg, Register ElementCountReg, Register TotalIterationsReg, bool IsMemcpy)
Adds logic in the loopBody MBB to generate MVE_VCTP, t2DoLoopDec and t2DoLoopEnd.
static SDValue PerformBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformBUILD_VECTORCombine - Target-specific dag combine xforms for ISD::BUILD_VECTOR.
static SDValue LowerVecReduceF(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformMinMaxCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformMinMaxCombine - Target-specific DAG combining for creating truncating saturates.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file defines the DenseMap class.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static Function * getFunction(Constant *C)
static bool isSigned(unsigned int Opcode)
const HexagonInstrInfo * TII
Module.h This file contains the declarations for the Module class.
std::pair< Value *, Value * > ShuffleOps
We are building a shuffle to create V, which is a sequence of insertelement, extractelement pairs.
static Value * LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP)
Emit the code to lower ctpop of V before the specified instruction IP.
loop Loop Strength Reduction
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
unsigned const TargetRegisterInfo * TRI
Promote Memory to Register
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
PowerPC Reduce CR logical Operation
const SmallVectorImpl< MachineOperand > & Cond
const MachineOperand & RHS
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
static SymbolRef::Type getType(const Symbol *Sym)
This file describes how to lower LLVM code to machine code.
static X86::CondCode getSwappedCondition(X86::CondCode CC)
Assuming the flags are set by MI(a,b), return the condition code if we modify the instructions such t...
static constexpr int Concat[]
bool getExactInverse(APFloat *inv) const
APInt bitcastToAPInt() const
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
unsigned logBase2() const
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
unsigned countr_one() const
Count the number of trailing one bits.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
An arbitrary precision integer that knows its signedness.
virtual const ARMBaseRegisterInfo & getRegisterInfo() const =0
const uint32_t * getSjLjDispatchPreservedMask(const MachineFunction &MF) const
const MCPhysReg * getCalleeSavedRegs(const MachineFunction *MF) const override
Code Generation virtual methods...
Register getFrameRegister(const MachineFunction &MF) const override
const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const override
const uint32_t * getTLSCallPreservedMask(const MachineFunction &MF) const
const uint32_t * getThisReturnPreservedMask(const MachineFunction &MF, CallingConv::ID) const
getThisReturnPreservedMask - Returns a call preserved mask specific to the case that 'returned' is on...
static ARMConstantPoolConstant * Create(const Constant *C, unsigned ID)
static ARMConstantPoolMBB * Create(LLVMContext &C, const MachineBasicBlock *mbb, unsigned ID, unsigned char PCAdj)
static ARMConstantPoolSymbol * Create(LLVMContext &C, StringRef s, unsigned ID, unsigned char PCAdj)
ARMConstantPoolValue - ARM specific constantpool value.
ARMFunctionInfo - This class is derived from MachineFunctionInfo and contains private ARM-specific in...
int getVarArgsFrameIndex() const
int getPromotedConstpoolIncrease() const
SmallPtrSet< const GlobalVariable *, 2 > & getGlobalsPromotedToConstantPool()
void setArgumentStackToRestore(unsigned v)
bool branchTargetEnforcement() const
unsigned createPICLabelUId()
void setPromotedConstpoolIncrease(int Sz)
bool isThumb1OnlyFunction() const
void setArgRegsSaveSize(unsigned s)
bool shouldSignReturnAddress() const
bool isCmseNSEntryFunction() const
void setReturnRegsCount(unsigned s)
void setVarArgsFrameIndex(int Index)
unsigned getArgRegsSaveSize() const
void markGlobalAsPromotedToConstantPool(const GlobalVariable *GV)
Indicate to the backend that GV has had its storage changed to inside a constant pool.
void setIsSplitCSR(bool s)
void setArgumentStackSize(unsigned size)
unsigned getArgumentStackSize() const
bool isTargetMachO() const
bool isTargetAEABI() const
bool supportsTailCall() const
const Triple & getTargetTriple() const
const ARMBaseInstrInfo * getInstrInfo() const override
bool isThumb1Only() const
bool hasFPARMv8Base() const
bool isTargetWindows() const
bool isGVIndirectSymbol(const GlobalValue *GV) const
True if the GV will be accessed via an indirect symbol.
const ARMTargetLowering * getTargetLowering() const override
bool isTargetDarwin() const
const ARMBaseRegisterInfo * getRegisterInfo() const override
bool isTargetAndroid() const
bool isTargetCOFF() const
bool isTargetGNUAEABI() const
bool isTargetWatchOS() const
unsigned getPreferBranchLogAlignment() const
bool useNEONForSinglePrecisionFP() const
const InstrItineraryData * getInstrItineraryData() const override
getInstrItins - Return the instruction itineraries based on subtarget selection.
bool isTargetWatchABI() const
bool hasAnyDataBarrier() const
bool isTargetDriverKit() const
bool allowsUnalignedMem() const
bool isTargetMuslAEABI() const
bool isTargetLinux() const
bool isTargetHardFloat() const
Align getDualLoadStoreAlignment() const
bool isReadOnly(const GlobalValue *GV) const
unsigned getMaxSupportedInterleaveFactor() const override
Get the maximum supported factor for interleaved memory accesses.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
unsigned getNumInterleavedAccesses(VectorType *VecTy, const DataLayout &DL) const
Returns the number of interleaved accesses that will be generated when lowering accesses of the given...
bool shouldInsertFencesForAtomic(const Instruction *I) const override
Whether AtomicExpandPass should automatically insert fences and reduce ordering for this atomic.
Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const override
Return the correct alignment for the current calling convention.
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const override
Examine constraint string and operand type and determine a weight value.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
isLegalAddressingMode - Return true if the addressing mode represented by AM is legal for this target...
const ARMSubtarget * getSubtarget() const
bool isLegalT2ScaledAddressingMode(const AddrMode &AM, EVT VT) const
bool isLegalT1ScaledAddressingMode(const AddrMode &AM, EVT VT) const
Returns true if the addressing mode representing by AM is legal for the Thumb1 target,...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPreIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mod...
bool shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize, Align &PrefAlign) const override
Return true if the pointer arguments to CI should be aligned by aligning the object whose address is ...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
ReplaceNodeResults - Replace the results of node with an illegal result type with new values built ou...
void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const override
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
bool isLegalAddImmediate(int64_t Imm) const override
isLegalAddImmediate - Return true if the specified immediate is legal add immediate,...
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
bool isFNegFree(EVT VT) const override
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
void finalizeLowering(MachineFunction &MF) const override
Execute target specific actions to finalize target lowering.
SDValue PerformMVETruncCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize=false) const override
isFPImmLegal - Returns true if the target can instruction select the specified FP immediate natively.
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
bool preferIncOfAddToSubOfNot(EVT VT) const override
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
Function * getSSPStackGuardCheck(const Module &M) const override
If the target has a standard stack protection check function that performs validation and error handl...
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const override
Returns how the given (atomic) store should be expanded by the IR-level AtomicExpand pass into.
void insertSSPDeclarations(Module &M) const override
Inserts necessary declarations for SSP (stack protection) purpose.
SDValue PerformIntrinsicCombine(SDNode *N, DAGCombinerInfo &DCI) const
PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
bool isDesirableToCommuteXorWithShift(const SDNode *N) const override
Return true if it is profitable to combine an XOR of a logical shift to create a logical shift of NOT...
bool ExpandInlineAsm(CallInst *CI) const override
This hook allows the target to expand an inline asm call to be explicit llvm code if it wants to.
SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const
PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
Value * createComplexDeinterleavingIR(IRBuilderBase &B, ComplexDeinterleavingOperation OperationType, ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB, Value *Accumulator=nullptr) const override
Create the IR node for the given complex deinterleaving operation.
bool isComplexDeinterleavingSupported() const override
Does this target support complex deinterleaving.
Value * getSDagStackGuard(const Module &M) const override
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
SDValue PerformMVEExtCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to fold a pair of shifts into a mask.
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &OriginalDemandedBits, const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
getSetCCResultType - Return the value type to use for ISD::SETCC.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Value * emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const override
Perform a store-conditional operation to Addr.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool isVarArg) const
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo) const override
createFastISel - This method returns a target specific FastISel object, or null if the target does no...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for this result type with this index.
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
getTgtMemIntrinsic - Represent NEON load and store intrinsics as MemIntrinsicNodes.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
isShuffleMaskLegal - Targets can use this to indicate that they only support some VECTOR_SHUFFLE oper...
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Returns true if it is beneficial to convert a load of a constant to just the constant itself.
bool useLoadStackGuardNode() const override
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const override
getRegClassFor - Return the register class that should be used for the specified value type.
std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const override
Return the largest legal super-reg register class of the register class for the specified type and it...
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vstN intrinsic.
ARMTargetLowering(const TargetMachine &TM, const ARMSubtarget &STI)
bool isComplexDeinterleavingOperationSupported(ComplexDeinterleavingOperation Operation, Type *Ty) const override
Does this target support complex deinterleaving with the given operation and type.
SDValue PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const
PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
Type * shouldConvertSplatType(ShuffleVectorInst *SVI) const override
Given a shuffle vector SVI representing a vector splat, return a new scalar type of size equal to SVI...
Value * emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr, AtomicOrdering Ord) const override
Perform a load-linked operation on Addr, returning a "Value *" with the corresponding pointee type.
Instruction * makeDMB(IRBuilderBase &Builder, ARM_MB::MemBOpt Domain) const
bool isLegalICmpImmediate(int64_t Imm) const override
isLegalICmpImmediate - Return true if the specified immediate is legal icmp immediate,...
const char * LowerXConstraint(EVT ConstraintVT) const override
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool isDesirableToTransformToIntegerOp(unsigned Opc, EVT VT) const override
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool isVarArg) const
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const override
allowsMisalignedMemoryAccesses - Returns true if the target allows unaligned memory accesses of the s...
bool isLegalInterleavedAccessType(unsigned Factor, FixedVectorType *VecTy, Align Alignment, const DataLayout &DL) const
Returns true if VecTy is a legal interleaved access type.
bool isVectorLoadExtDesirable(SDValue ExtVal) const override
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx, unsigned &Cost) const override
Return true if the target can combine store(extractelement VectorTy,Idx).
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vldN intrinsic.
bool useSoftFloat() const override
bool alignLoopsWithOptSize() const override
Should loops be aligned even when the function is marked OptSize (but not MinSize).
SDValue PerformCMOVToBFICombine(SDNode *N, SelectionDAG &DAG) const
bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override
Return true if a truncation from FromTy to ToTy is permitted when deciding whether a call is in tail ...
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
LowerAsmOperandForConstraint - Lower the specified operand into the Ops vector.
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const override
Returns true if an argument of type Ty needs to be passed in a contiguous block of registers in calli...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const override
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPostIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mo...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
bool isFloatingPointOperation() const
This class holds the attributes for a function, its return value, and its parameters.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
LLVM Basic Block Representation.
The address of a basic block.
static BranchProbability getZero()
A "pseudo-class" with methods for operating on BUILD_VECTORs.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, uint32_t BitWidth) const
If this is a constant FP splat and the splatted constant FP is an exact power or 2,...
CCState - This class holds information needed while lowering arguments and return values.
void getInRegsParamInfo(unsigned InRegsParamRecordIndex, unsigned &BeginReg, unsigned &EndReg) const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
static bool resultsCompatible(CallingConv::ID CalleeCC, CallingConv::ID CallerCC, MachineFunction &MF, LLVMContext &C, const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn CalleeFn, CCAssignFn CallerFn)
Returns true if the results of the two calling conventions are compatible.
bool CheckReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
CheckReturn - Analyze the return values of a function, returning true if the return can be performed ...
void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
void rewindByValRegsInfo()
unsigned getInRegsParamsProcessed() const
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
unsigned getInRegsParamsCount() const
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
int64_t getLocMemOffset() const
unsigned getValNo() const
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
Value * getCalledOperand() const
AttributeList getAttributes() const
Return the parameter attributes for this call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
const APFloat & getValueAPF() const
ConstantFP - Floating Point Values [float, double].
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
int64_t getSExtValue() const
This is an important base class in LLVM.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
bool isLittleEndian() const
Layout endianness...
MaybeAlign getStackAlignment() const
Returns the natural stack alignment, or MaybeAlign() if one wasn't specified.
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Align getPreferredAlign(const GlobalVariable *GV) const
Returns the preferred alignment of the specified global.
StringRef getPrivateGlobalPrefix() const
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Diagnostic information for unsupported feature in backend.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Type * getParamType(unsigned i) const
Parameter type accessors.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
const Function & getFunction() const
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool hasStructRetAttr() const
Determine if the function returns a structure through first or second pointer argument.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
const GlobalValue * getGlobal() const
bool hasExternalWeakLinkage() const
bool hasDLLImportStorageClass() const
bool isStrongDefinitionForLinker() const
Returns true if this global's definition will be the one chosen by the linker.
@ InternalLinkage
Rename collisions when linking (static functions).
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
TargetInstrInfo overrides.
Common base class shared among various IRBuilders.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
std::optional< unsigned > getOperandCycle(unsigned ItinClassIndx, unsigned OperandIdx) const
Return the cycle for the given class and operand.
bool isEmpty() const
Returns true if there are no itineraries.
bool hasAtomicStore() const LLVM_READONLY
Return true if this atomic instruction stores to memory.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
static bool LowerToByteSwap(CallInst *CI)
Try to replace a call instruction with a call to a bswap intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Describe properties that are true of each instruction in the target description file.
unsigned getSchedClass() const
Return the scheduling class for this instruction.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
ArrayRef< MCOperandInfo > operands() const
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specified operand constraint if it is present.
Wrapper class representing physical registers. Should be passed by value.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isInteger() const
Return true if this is an integer or a vector integer type.
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
static auto integer_valuetypes()
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
static auto fp_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
bool isEHPad() const
Returns true if the block is a landing pad.
MachineBasicBlock * getFallThrough(bool JumpToFallThrough=true)
Return the fallthrough block if the block can implicitly transfer control to the block after it by fa...
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
void setCallFrameSize(unsigned N)
Set the call frame size on entry to this basic block.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
bool canFallThrough()
Return true if the block can implicitly transfer control to the block after it by falling off the end...
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
pred_iterator pred_begin()
MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
iterator_range< succ_iterator > successors()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
void moveAfter(MachineBasicBlock *NewBefore)
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
void computeMaxCallFrameSize(MachineFunction &MF, std::vector< MachineBasicBlock::iterator > *FrameSDOps=nullptr)
Computes the maximum size of a callframe.
void setAdjustsStack(bool V)
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool hasVAStart() const
Returns true if the function calls the llvm.va_start intrinsic.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
int getFunctionContextIndex() const
Return the index for the function context object.
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & reset(Property P)
unsigned getFunctionNumber() const
getFunctionNumber - Return a unique ID for the current function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
void push_back(MachineBasicBlock *MBB)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
const MachineFunctionProperties & getProperties() const
Get the function properties.
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addJumpTableIndex(unsigned Idx, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
bool readsRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr reads the specified register.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
const MachineOperand & getOperand(unsigned i) const
unsigned createJumpTableIndex(const std::vector< MachineBasicBlock * > &DestBBs)
createJumpTableIndex - Create a new jump table.
@ EK_Inline
EK_Inline - Jump table entries are emitted inline at their point of use.
@ EK_BlockAddress
EK_BlockAddress - Each entry is a plain address of block, e.g.: .word LBB123.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
void setIsDef(bool Val=true)
Change a def to a use, or a use to a def.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class is used to represent an MLOAD node.
This class is used to represent an MSTORE node.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
This is an abstract virtual class for memory operations.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
virtual void print(raw_ostream &OS, const Module *M) const
print - Print out the internal state of the pass.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< use_iterator > uses()
size_t use_size() const
Return the number of uses of this node.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
const APInt & getConstantOperandAPInt(unsigned Num) const
Helper method returns the APInt of a ConstantSDNode operand.
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Return true if the type of the node type undefined.
void setFlags(SDNodeFlags NewFlags)
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getStackArgumentTokenFactor(SDValue Chain)
Compute a TokenFactor to force all the incoming stack arguments to be loaded from the stack.
const TargetSubtargetInfo & getSubtarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
SDValue getRegister(Register Reg, EVT VT)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
void addCallSiteInfo(const SDNode *Node, CallSiteInfo &&CallInfo)
Set CallSiteInfo to be associated with Node.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
This instruction constructs a fixed permutation of two input vectors.
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getMaskElt(unsigned Idx) const
int getSplatIndex() const
ArrayRef< int > getMask() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
StringRef - Represent a constant reference to a string, i.e.
const unsigned char * bytes_end() const
constexpr size_t size() const
size - Get the string size.
const unsigned char * bytes_begin() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
void setMaxDivRemBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum div/rem the backend supports.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
void setCmpLibcallCC(RTLIB::Libcall Call, ISD::CondCode CC)
Override the default CondCode to be used to test the result of the comparison libcall against zero.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
void setMinStackArgumentAlignment(Align Alignment)
Set the minimum stack alignment of an argument.
const TargetMachine & getTargetMachine() const
void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC)
Set the CallingConv that should be used for the specified libcall.
void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked load does or does not work with the specified type and ind...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Sched::Preference getSchedulingPreference() const
Return target scheduling preference.
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
virtual unsigned getMaxSupportedInterleaveFactor() const
Get the maximum supported factor for interleaved memory accesses.
void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked store does or does not work with the specified type and in...
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
virtual std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const
Return the largest legal super-reg register class of the register class for the specified type and it...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
virtual void insertSSPDeclarations(Module &M) const
Inserts necessary declarations for SSP (stack protection) purpose.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
bool isPositionIndependent() const
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef< int > Mask, SelectionDAG &DAG) const
Tries to build a legal vector shuffle using the provided parameters or equivalent variations.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
bool isConstTrueVal(SDValue N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
const Triple & getTargetTriple() const
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
virtual const TargetSubtargetInfo * getSubtargetImpl(const Function &) const
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
ObjectFormatType getObjectFormat() const
Get the object format for this triple.
bool isOSVersionLT(unsigned Major, unsigned Minor=0, unsigned Micro=0) const
Helper function for doing comparisons against version numbers included in the target triple.
bool isWindowsMSVCEnvironment() const
Checks if the environment could be MSVC.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
static IntegerType * getInt32Ty(LLVMContext &C)
bool isPointerTy() const
True if this is an instance of PointerType.
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt8Ty(LLVMContext &C)
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
static IntegerType * getInt16Ty(LLVMContext &C)
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
static IntegerType * getInt64Ty(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
const Use & getOperandUse(unsigned i) const
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
Base class of all SIMD vector types.
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
const ParentTy * getParent() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
static CondCodes getOppositeCondition(CondCodes CC)
@ SECREL
Thread Pointer Offset.
@ SBREL
Section Relative (Windows TLS)
@ GOTTPOFF
Global Offset Table, PC Relative.
@ TPOFF
Global Offset Table, Thread Pointer Offset.
TOF
Target Operand Flag enum.
@ MO_NONLAZY
MO_NONLAZY - This is an independent flag, on a symbol operand "FOO" it represents a symbol which,...
@ MO_SBREL
MO_SBREL - On a symbol operand, this represents a static base relative relocation.
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand, this represents that the reference to the symbol is for an import...
@ MO_GOT
MO_GOT - On a symbol operand, this represents a GOT relative relocation.
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
static ShiftOpc getShiftOpcForNode(unsigned Opcode)
int getSOImmVal(unsigned Arg)
getSOImmVal - Given a 32-bit immediate, if it is something that can fit into an shifter_operand immed...
int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
uint64_t decodeVMOVModImm(unsigned ModImm, unsigned &EltBits)
decodeVMOVModImm - Decode a NEON/MVE modified immediate value into the element value and the element ...
unsigned getAM2Offset(unsigned AM2Opc)
bool isThumbImmShiftedVal(unsigned V)
isThumbImmShiftedVal - Return true if the specified value can be obtained by left shifting a 8-bit im...
int getT2SOImmVal(unsigned Arg)
getT2SOImmVal - Given a 32-bit immediate, if it is something that can fit into a Thumb-2 shifter_oper...
unsigned createVMOVModImm(unsigned OpCmode, unsigned Val)
int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
int getFP16Imm(const APInt &Imm)
getFP16Imm - Return an 8-bit floating-point version of the 16-bit floating-point value.
unsigned getSORegOpc(ShiftOpc ShOp, unsigned Imm)
int getFP32FP16Imm(const APInt &Imm)
If this is a FP16Imm encoded as a fp32 value, return the 8-bit encoding for it.
AddrOpc getAM2Op(unsigned AM2Opc)
bool isBitFieldInvertedMask(unsigned v)
const unsigned FPStatusBits
const unsigned FPReservedBits
const unsigned RoundingBitsPos
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Swift
Calling convention for Swift.
@ ARM_APCS
ARM Procedure Calling Standard (obsolete, but still used on some targets).
@ CFGuard_Check
Special calling convention on Windows for calling the Control Guard Check ICall funtion.
@ PreserveMost
Used for runtime calls that preserves most registers.
@ ARM_AAPCS
ARM Architecture Procedure Calling Standard calling convention (aka EABI).
@ CXX_FAST_TLS
Used for access functions.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
@ ARM_AAPCS_VFP
Same as ARM_AAPCS, but uses hard floating point ABI.
@ C
The default llvm calling convention, compatible with C.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ ADD
Simple integer binary arithmetic operators.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ SSUBO
Same for subtraction.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ SMULO
Same for multiplication.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
static const int LAST_INDEXED_MODE
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Create or insert an LLVM Function declaration for an intrinsic, and return it.
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
FunctionAddr VTableAddr Value
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool CC_ARM_APCS_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool RetCC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns true if Val1 has a lower Constant Materialization Cost than Val2.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
bool isStrongerThanMonotonic(AtomicOrdering AO)
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
FunctionAddr VTableAddr uintptr_t uintptr_t Int32Ty
constexpr bool isMask_32(uint32_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
bool FastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
bool RetCC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
void shuffle(Iterator first, Iterator last, RNG &&g)
static std::array< MachineOperand, 2 > predOps(ARMCC::CondCodes Pred, unsigned PredReg=0)
Get the operands corresponding to the given Pred value.
bool CC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
bool CC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
constexpr bool has_single_bit(T Value) noexcept
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
bool RetCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_ARM_Win32_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
ComplexDeinterleavingOperation
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
FunctionAddr VTableAddr Count
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
ComplexDeinterleavingRotation
FunctionAddr VTableAddr uintptr_t uintptr_t Data
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
unsigned ConstantMaterializationCost(unsigned Val, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns the number of instructions required to materialize the given constant in a register,...
@ Mul
Product of integers.
@ And
Bitwise or logical AND of integers.
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
bool RetFastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
static MachineOperand t1CondCodeOp(bool isDead=false)
Get the operand corresponding to the conditional code result for Thumb1.
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
static MachineOperand condCodeOp(unsigned CCReg=0)
Get the operand corresponding to the conditional code result.
bool isVREVMask(ArrayRef< int > M, EVT VT, unsigned BlockSize)
isVREVMask - Check if a vector shuffle corresponds to a VREV instruction with the specified blocksize...
auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
unsigned gettBLXrOpcode(const MachineFunction &MF)
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
constexpr bool isShiftedUInt(uint64_t x)
Checks if a unsigned integer is an N bit number shifted left by S.
unsigned convertAddSubFlagsOpcode(unsigned OldOpc)
Map pseudo instructions that imply an 'S' bit onto real opcodes.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
static const unsigned PerfectShuffleTable[6561+1]
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Load/store instruction that can be merged with a base address update.
SDNode * N
Instruction that updates a pointer.
unsigned ConstInc
Pointer increment value if it is a constant, or 0 otherwise.
SDValue Inc
Pointer increment operand.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
static constexpr roundingMode rmTowardZero
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
bool isUnknown() const
Returns true if we don't know any bits.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits add(const KnownBits &LHS, const KnownBits &RHS, bool NSW=false, bool NUW=false)
Compute knownbits resulting from addition of LHS and RHS.
static KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoSignedZeros() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This contains information for each constraint that we are lowering.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setInRegister(bool Value=true)
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setZExtResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setSExtResult(bool Value=true)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)